{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use TRPO to Play Acrobot-v1\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import scipy.signal as signal\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:56:31 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "11:56:31 [INFO] action_space: Discrete(3)\n",
      "11:56:31 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "11:56:31 [INFO] reward_range: (-inf, inf)\n",
      "11:56:31 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "11:56:31 [INFO] _max_episode_steps: 500\n",
      "11:56:31 [INFO] _elapsed_steps: None\n",
      "11:56:31 [INFO] id: Acrobot-v1\n",
      "11:56:31 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv\n",
      "11:56:31 [INFO] reward_threshold: -100.0\n",
      "11:56:31 [INFO] nondeterministic: False\n",
      "11:56:31 [INFO] max_episode_steps: 500\n",
      "11:56:31 [INFO] _kwargs: {}\n",
      "11:56:31 [INFO] _env_name: Acrobot\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PPOReplayer:\n",
    "    def __init__(self):\n",
    "        self.fields = ['state', 'action', 'prob', 'advantage', 'return']\n",
    "        self.memory = pd.DataFrame(columns=self.fields)\n",
    "\n",
    "    def store(self, df):\n",
    "        self.memory = pd.concat([self.memory, df[self.fields]], ignore_index=True)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.memory.shape[0], size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conjugate_gradient(f, b, iter_count=10, epsilon=1e-12, tol=1e-6):\n",
    "    x = b * 0.\n",
    "    r = tf.identity(b)\n",
    "    p = tf.identity(b)\n",
    "    rho = tf.reduce_sum(r * r)\n",
    "    for i in range(iter_count):\n",
    "        z = f(p)\n",
    "        alpha = rho / (tf.reduce_sum(p * z) + epsilon)\n",
    "        x += alpha * p\n",
    "        r -= alpha * z\n",
    "        rho_new = tf.reduce_sum(r * r)\n",
    "        p = r + (rho_new / rho) * p\n",
    "        rho = rho_new\n",
    "        if rho < tol:\n",
    "            break\n",
    "    return x, f(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TRPOAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = PPOReplayer()\n",
    "        self.trajectory = []\n",
    "\n",
    "        self.max_kl = 0.01\n",
    "        self.actor_net = self.build_net(hidden_sizes=[100,],\n",
    "                output_size=self.action_n, output_activation=nn.softmax)\n",
    "        self.critic_net = self.build_net(hidden_sizes=[100,],\n",
    "                learning_rate=0.002)\n",
    "\n",
    "    def build_net(self, input_size=None, hidden_sizes=None, output_size=1,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            self.save_trajectory_to_replayer()\n",
    "            if len(self.replayer.memory) >= 1000:\n",
    "                for batch in range(5): # learn multiple times\n",
    "                    self.learn()\n",
    "                self.replayer = PPOReplayer() # reset replayer after the agent changes itself\n",
    "\n",
    "    def save_trajectory_to_replayer(self):\n",
    "        df = pd.DataFrame(\n",
    "                np.array(self.trajectory, dtype=object).reshape(-1, 4),\n",
    "                columns=['state', 'reward', 'done', 'action'], dtype=object)\n",
    "        states = np.stack(df['state'])\n",
    "        df['v'] = self.critic_net.predict(states)\n",
    "        pis = self.actor_net.predict(states)\n",
    "        df['prob'] = [pi[action] for pi, action in zip(pis, df['action'])]\n",
    "        df['next_v'] = df['v'].shift(-1).fillna(0.)\n",
    "        df['u'] = df['reward'] + self.gamma * df['next_v']\n",
    "        df['delta'] = df['u'] - df['v']\n",
    "        df['advantage'] = signal.lfilter([1.,], [1., -self.gamma],\n",
    "                df['delta'][::-1])[::-1]\n",
    "        df['return'] = signal.lfilter([1.,], [1., -self.gamma],\n",
    "                df['reward'][::-1])[::-1]\n",
    "        self.replayer.store(df)\n",
    "\n",
    "    def learn(self):\n",
    "        states, actions, old_pis, advantages, returns = \\\n",
    "                self.replayer.sample(size=64)\n",
    "        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)\n",
    "        action_tensor = tf.convert_to_tensor(actions, dtype=tf.int32)\n",
    "        old_pi_tensor = tf.convert_to_tensor(old_pis, dtype=tf.float32)\n",
    "        advantage_tensor = tf.convert_to_tensor(advantages, dtype=tf.float32)\n",
    "\n",
    "        # train actor\n",
    "        # ... calculate first order gradient of KL divergence\n",
    "        with tf.GradientTape() as tape:\n",
    "            all_pi_tensor = self.actor_net(state_tensor)\n",
    "            pi_tensor = tf.gather(all_pi_tensor, action_tensor, batch_dims=1)\n",
    "            surrogate_tensor = (pi_tensor / old_pi_tensor) * advantage_tensor\n",
    "        actor_grads = tape.gradient(surrogate_tensor, self.actor_net.variables)\n",
    "        loss_grad = tf.concat([tf.reshape(grad, (-1,)) for grad in actor_grads], axis=0)\n",
    "\n",
    "        # ... calculate conjugate gradient: Fx = g\n",
    "        def f(x): # calculate Fx\n",
    "            with tf.GradientTape() as tape2: # tape for 2nd-order gradient\n",
    "                with tf.GradientTape() as tape1: # tape for 1st-order gradient\n",
    "                    prob_tensor = self.actor_net(state_tensor)\n",
    "                    prob_old_tensor = tf.stop_gradient(prob_tensor)\n",
    "                    kld_tensor = tf.reduce_sum(prob_old_tensor * (tf.math.log(\n",
    "                            prob_old_tensor) - tf.math.log(prob_tensor)), axis=1)\n",
    "                    kld_loss_tensor = tf.reduce_mean(kld_tensor)\n",
    "                grads = tape1.gradient(kld_loss_tensor, self.actor_net.variables)\n",
    "                flatten_grad_tensor = tf.concat(\n",
    "                        [tf.reshape(grad, (-1,)) for grad in grads], axis=-1)\n",
    "                grad_matmul_x = tf.tensordot(flatten_grad_tensor, x, axes=[[-1], [-1]])\n",
    "            grad_grads = tape2.gradient(grad_matmul_x, self.actor_net.variables)\n",
    "            flatten_grad_grad = tf.stop_gradient(tf.concat(\n",
    "                    [tf.reshape(grad_grad, (-1,)) for grad_grad in grad_grads], axis=-1))\n",
    "            fx = flatten_grad_grad + x * 1e-2\n",
    "            return fx\n",
    "        x, fx = conjugate_gradient(f, loss_grad)\n",
    "\n",
    "        # ... calculate natural gradient\n",
    "        natural_gradient_tensor = tf.sqrt(2 * self.max_kl / tf.reduce_sum(fx * x)) * x\n",
    "        # ....... refactor the flatten gradient into un-flatten version\n",
    "        flatten_natural_gradient = natural_gradient_tensor.numpy()\n",
    "        natural_grads = []\n",
    "        begin = 0\n",
    "        for weight in self.actor_net.get_weights():\n",
    "            end = begin + weight.size\n",
    "            natural_grad = flatten_natural_gradient[begin:end].reshape(weight.shape)\n",
    "            natural_grads.append(natural_grad)\n",
    "            begin = end\n",
    "\n",
    "        # ... line search\n",
    "        old_weights = self.actor_net.get_weights()\n",
    "        expected_improve = tf.reduce_sum(loss_grad * natural_gradient_tensor).numpy()\n",
    "        for learning_step in [0.,] + [.5 ** j for j in range(10)]:\n",
    "            self.actor_net.set_weights([weight + learning_step * grad\n",
    "                    for weight, grad in zip(old_weights, natural_grads)])\n",
    "            all_pi_tensor = self.actor_net(state_tensor)\n",
    "            new_pi_tensor = tf.gather(all_pi_tensor, action_tensor[:, np.newaxis], axis=1)[:, 0]\n",
    "            new_pi_tensor = tf.stop_gradient(new_pi_tensor)\n",
    "            surrogate_tensor = (new_pi_tensor / pi_tensor) * advantage_tensor\n",
    "            objective = tf.reduce_sum(surrogate_tensor).numpy()\n",
    "            if np.isclose(learning_step, 0.):\n",
    "                old_objective = objective\n",
    "            else:\n",
    "                if objective - old_objective > 0.1 * expected_improve * learning_step:\n",
    "                    break # success, keep the weight\n",
    "        else:\n",
    "            self.actor_net.set_weights(old_weights)\n",
    "\n",
    "        # train critic\n",
    "        self.critic_net.fit(states, returns, verbose=0)\n",
    "\n",
    "\n",
    "agent = TRPOAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:56:32 [INFO] ==== train ====\n",
      "11:57:06 [INFO] NumExpr defaulting to 8 threads.\n",
      "11:57:06 [DEBUG] train episode 0: reward = -500.00, steps = 500\n",
      "11:57:41 [DEBUG] train episode 1: reward = -500.00, steps = 500\n",
      "11:58:14 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "11:58:39 [DEBUG] train episode 3: reward = -386.00, steps = 387\n",
      "11:59:14 [DEBUG] train episode 4: reward = -500.00, steps = 500\n",
      "11:59:42 [DEBUG] train episode 5: reward = -425.00, steps = 426\n",
      "12:00:09 [DEBUG] train episode 6: reward = -399.00, steps = 400\n",
      "12:00:39 [DEBUG] train episode 7: reward = -427.00, steps = 428\n",
      "12:00:59 [DEBUG] train episode 8: reward = -299.00, steps = 300\n",
      "12:01:23 [DEBUG] train episode 9: reward = -369.00, steps = 370\n",
      "12:01:58 [DEBUG] train episode 10: reward = -500.00, steps = 500\n",
      "12:02:21 [DEBUG] train episode 11: reward = -341.00, steps = 342\n",
      "12:02:48 [DEBUG] train episode 12: reward = -405.00, steps = 406\n",
      "12:03:15 [DEBUG] train episode 13: reward = -394.00, steps = 395\n",
      "12:03:36 [DEBUG] train episode 14: reward = -306.00, steps = 307\n",
      "12:04:09 [DEBUG] train episode 15: reward = -500.00, steps = 500\n",
      "12:04:30 [DEBUG] train episode 16: reward = -310.00, steps = 311\n",
      "12:04:54 [DEBUG] train episode 17: reward = -356.00, steps = 357\n",
      "12:05:11 [DEBUG] train episode 18: reward = -259.00, steps = 260\n",
      "12:05:32 [DEBUG] train episode 19: reward = -315.00, steps = 316\n",
      "12:06:02 [DEBUG] train episode 20: reward = -420.00, steps = 421\n",
      "12:06:34 [DEBUG] train episode 21: reward = -500.00, steps = 500\n",
      "12:06:58 [DEBUG] train episode 22: reward = -351.00, steps = 352\n",
      "12:07:33 [DEBUG] train episode 23: reward = -500.00, steps = 500\n",
      "12:07:53 [DEBUG] train episode 24: reward = -308.00, steps = 309\n",
      "12:08:25 [DEBUG] train episode 25: reward = -486.00, steps = 487\n",
      "12:08:38 [DEBUG] train episode 26: reward = -195.00, steps = 196\n",
      "12:09:11 [DEBUG] train episode 27: reward = -479.00, steps = 480\n",
      "12:09:44 [DEBUG] train episode 28: reward = -500.00, steps = 500\n",
      "12:10:04 [DEBUG] train episode 29: reward = -300.00, steps = 301\n",
      "12:10:22 [DEBUG] train episode 30: reward = -265.00, steps = 266\n",
      "12:10:43 [DEBUG] train episode 31: reward = -307.00, steps = 308\n",
      "12:11:11 [DEBUG] train episode 32: reward = -427.00, steps = 428\n",
      "12:11:31 [DEBUG] train episode 33: reward = -272.00, steps = 273\n",
      "12:11:55 [DEBUG] train episode 34: reward = -375.00, steps = 376\n",
      "12:12:18 [DEBUG] train episode 35: reward = -337.00, steps = 338\n",
      "12:12:42 [DEBUG] train episode 36: reward = -339.00, steps = 340\n",
      "12:13:09 [DEBUG] train episode 37: reward = -419.00, steps = 420\n",
      "12:13:24 [DEBUG] train episode 38: reward = -215.00, steps = 216\n",
      "12:13:37 [DEBUG] train episode 39: reward = -200.00, steps = 201\n",
      "12:13:53 [DEBUG] train episode 40: reward = -215.00, steps = 216\n",
      "12:14:10 [DEBUG] train episode 41: reward = -252.00, steps = 253\n",
      "12:14:24 [DEBUG] train episode 42: reward = -223.00, steps = 224\n",
      "12:14:33 [DEBUG] train episode 43: reward = -123.00, steps = 124\n",
      "12:14:52 [DEBUG] train episode 44: reward = -298.00, steps = 299\n",
      "12:15:07 [DEBUG] train episode 45: reward = -196.00, steps = 197\n",
      "12:15:20 [DEBUG] train episode 46: reward = -198.00, steps = 199\n",
      "12:15:41 [DEBUG] train episode 47: reward = -315.00, steps = 316\n",
      "12:15:57 [DEBUG] train episode 48: reward = -244.00, steps = 245\n",
      "12:16:15 [DEBUG] train episode 49: reward = -249.00, steps = 250\n",
      "12:16:28 [DEBUG] train episode 50: reward = -193.00, steps = 194\n",
      "12:16:41 [DEBUG] train episode 51: reward = -199.00, steps = 200\n",
      "12:16:51 [DEBUG] train episode 52: reward = -141.00, steps = 142\n",
      "12:17:09 [DEBUG] train episode 53: reward = -286.00, steps = 287\n",
      "12:17:20 [DEBUG] train episode 54: reward = -163.00, steps = 164\n",
      "12:17:38 [DEBUG] train episode 55: reward = -250.00, steps = 251\n",
      "12:17:51 [DEBUG] train episode 56: reward = -187.00, steps = 188\n",
      "12:18:03 [DEBUG] train episode 57: reward = -178.00, steps = 179\n",
      "12:18:10 [DEBUG] train episode 58: reward = -104.00, steps = 105\n",
      "12:18:20 [DEBUG] train episode 59: reward = -161.00, steps = 162\n",
      "12:18:30 [DEBUG] train episode 60: reward = -145.00, steps = 146\n",
      "12:18:43 [DEBUG] train episode 61: reward = -193.00, steps = 194\n",
      "12:19:00 [DEBUG] train episode 62: reward = -236.00, steps = 237\n",
      "12:19:11 [DEBUG] train episode 63: reward = -165.00, steps = 166\n",
      "12:19:27 [DEBUG] train episode 64: reward = -244.00, steps = 245\n",
      "12:19:41 [DEBUG] train episode 65: reward = -202.00, steps = 203\n",
      "12:19:50 [DEBUG] train episode 66: reward = -133.00, steps = 134\n",
      "12:19:58 [DEBUG] train episode 67: reward = -124.00, steps = 125\n",
      "12:20:09 [DEBUG] train episode 68: reward = -140.00, steps = 141\n",
      "12:20:19 [DEBUG] train episode 69: reward = -142.00, steps = 143\n",
      "12:20:30 [DEBUG] train episode 70: reward = -164.00, steps = 165\n",
      "12:20:45 [DEBUG] train episode 71: reward = -223.00, steps = 224\n",
      "12:21:05 [DEBUG] train episode 72: reward = -295.00, steps = 296\n",
      "12:21:16 [DEBUG] train episode 73: reward = -154.00, steps = 155\n",
      "12:21:27 [DEBUG] train episode 74: reward = -142.00, steps = 143\n",
      "12:21:39 [DEBUG] train episode 75: reward = -177.00, steps = 178\n",
      "12:21:53 [DEBUG] train episode 76: reward = -184.00, steps = 185\n",
      "12:22:03 [DEBUG] train episode 77: reward = -139.00, steps = 140\n",
      "12:22:18 [DEBUG] train episode 78: reward = -236.00, steps = 237\n",
      "12:22:34 [DEBUG] train episode 79: reward = -229.00, steps = 230\n",
      "12:22:45 [DEBUG] train episode 80: reward = -146.00, steps = 147\n",
      "12:23:01 [DEBUG] train episode 81: reward = -243.00, steps = 244\n",
      "12:23:12 [DEBUG] train episode 82: reward = -163.00, steps = 164\n",
      "12:23:25 [DEBUG] train episode 83: reward = -195.00, steps = 196\n",
      "12:23:34 [DEBUG] train episode 84: reward = -119.00, steps = 120\n",
      "12:23:49 [DEBUG] train episode 85: reward = -235.00, steps = 236\n",
      "12:24:11 [DEBUG] train episode 86: reward = -303.00, steps = 304\n",
      "12:24:22 [DEBUG] train episode 87: reward = -164.00, steps = 165\n",
      "12:24:33 [DEBUG] train episode 88: reward = -173.00, steps = 174\n",
      "12:24:43 [DEBUG] train episode 89: reward = -137.00, steps = 138\n",
      "12:24:53 [DEBUG] train episode 90: reward = -148.00, steps = 149\n",
      "12:25:05 [DEBUG] train episode 91: reward = -187.00, steps = 188\n",
      "12:25:13 [DEBUG] train episode 92: reward = -106.00, steps = 107\n",
      "12:25:26 [DEBUG] train episode 93: reward = -185.00, steps = 186\n",
      "12:25:41 [DEBUG] train episode 94: reward = -227.00, steps = 228\n",
      "12:25:51 [DEBUG] train episode 95: reward = -143.00, steps = 144\n",
      "12:26:02 [DEBUG] train episode 96: reward = -162.00, steps = 163\n",
      "12:26:13 [DEBUG] train episode 97: reward = -163.00, steps = 164\n",
      "12:26:23 [DEBUG] train episode 98: reward = -156.00, steps = 157\n",
      "12:26:34 [DEBUG] train episode 99: reward = -143.00, steps = 144\n",
      "12:26:42 [DEBUG] train episode 100: reward = -120.00, steps = 121\n",
      "12:26:51 [DEBUG] train episode 101: reward = -128.00, steps = 129\n",
      "12:27:03 [DEBUG] train episode 102: reward = -177.00, steps = 178\n",
      "12:27:11 [DEBUG] train episode 103: reward = -125.00, steps = 126\n",
      "12:27:27 [DEBUG] train episode 104: reward = -242.00, steps = 243\n",
      "12:27:37 [DEBUG] train episode 105: reward = -155.00, steps = 156\n",
      "12:27:50 [DEBUG] train episode 106: reward = -177.00, steps = 178\n",
      "12:27:56 [DEBUG] train episode 107: reward = -83.00, steps = 84\n",
      "12:28:08 [DEBUG] train episode 108: reward = -181.00, steps = 182\n",
      "12:28:18 [DEBUG] train episode 109: reward = -148.00, steps = 149\n",
      "12:28:27 [DEBUG] train episode 110: reward = -133.00, steps = 134\n",
      "12:28:35 [DEBUG] train episode 111: reward = -128.00, steps = 129\n",
      "12:28:42 [DEBUG] train episode 112: reward = -97.00, steps = 98\n",
      "12:28:51 [DEBUG] train episode 113: reward = -132.00, steps = 133\n",
      "12:29:01 [DEBUG] train episode 114: reward = -130.00, steps = 131\n",
      "12:29:13 [DEBUG] train episode 115: reward = -171.00, steps = 172\n",
      "12:29:22 [DEBUG] train episode 116: reward = -140.00, steps = 141\n",
      "12:29:32 [DEBUG] train episode 117: reward = -148.00, steps = 149\n",
      "12:29:42 [DEBUG] train episode 118: reward = -153.00, steps = 154\n",
      "12:29:53 [DEBUG] train episode 119: reward = -151.00, steps = 152\n",
      "12:30:05 [DEBUG] train episode 120: reward = -186.00, steps = 187\n",
      "12:30:18 [DEBUG] train episode 121: reward = -169.00, steps = 170\n",
      "12:30:30 [DEBUG] train episode 122: reward = -184.00, steps = 185\n",
      "12:30:41 [DEBUG] train episode 123: reward = -157.00, steps = 158\n",
      "12:30:50 [DEBUG] train episode 124: reward = -143.00, steps = 144\n",
      "12:31:05 [DEBUG] train episode 125: reward = -213.00, steps = 214\n",
      "12:31:16 [DEBUG] train episode 126: reward = -167.00, steps = 168\n",
      "12:31:32 [DEBUG] train episode 127: reward = -212.00, steps = 213\n",
      "12:31:39 [DEBUG] train episode 128: reward = -115.00, steps = 116\n",
      "12:31:49 [DEBUG] train episode 129: reward = -145.00, steps = 146\n",
      "12:32:00 [DEBUG] train episode 130: reward = -161.00, steps = 162\n",
      "12:32:10 [DEBUG] train episode 131: reward = -153.00, steps = 154\n",
      "12:32:33 [DEBUG] train episode 132: reward = -343.00, steps = 344\n",
      "12:32:47 [DEBUG] train episode 133: reward = -185.00, steps = 186\n",
      "12:33:00 [DEBUG] train episode 134: reward = -194.00, steps = 195\n",
      "12:33:11 [DEBUG] train episode 135: reward = -160.00, steps = 161\n",
      "12:33:25 [DEBUG] train episode 136: reward = -223.00, steps = 224\n",
      "12:33:40 [DEBUG] train episode 137: reward = -211.00, steps = 212\n",
      "12:33:56 [DEBUG] train episode 138: reward = -237.00, steps = 238\n",
      "12:34:07 [DEBUG] train episode 139: reward = -163.00, steps = 164\n",
      "12:34:22 [DEBUG] train episode 140: reward = -213.00, steps = 214\n",
      "12:34:37 [DEBUG] train episode 141: reward = -226.00, steps = 227\n",
      "12:34:46 [DEBUG] train episode 142: reward = -140.00, steps = 141\n",
      "12:34:57 [DEBUG] train episode 143: reward = -169.00, steps = 170\n",
      "12:35:09 [DEBUG] train episode 144: reward = -149.00, steps = 150\n",
      "12:35:17 [DEBUG] train episode 145: reward = -128.00, steps = 129\n",
      "12:35:26 [DEBUG] train episode 146: reward = -130.00, steps = 131\n",
      "12:35:37 [DEBUG] train episode 147: reward = -159.00, steps = 160\n",
      "12:35:45 [DEBUG] train episode 148: reward = -125.00, steps = 126\n",
      "12:35:56 [DEBUG] train episode 149: reward = -156.00, steps = 157\n",
      "12:36:03 [DEBUG] train episode 150: reward = -102.00, steps = 103\n",
      "12:36:09 [DEBUG] train episode 151: reward = -88.00, steps = 89\n",
      "12:36:21 [DEBUG] train episode 152: reward = -151.00, steps = 152\n",
      "12:36:28 [DEBUG] train episode 153: reward = -108.00, steps = 109\n",
      "12:36:38 [DEBUG] train episode 154: reward = -156.00, steps = 157\n",
      "12:36:48 [DEBUG] train episode 155: reward = -135.00, steps = 136\n",
      "12:36:57 [DEBUG] train episode 156: reward = -137.00, steps = 138\n",
      "12:37:05 [DEBUG] train episode 157: reward = -120.00, steps = 121\n",
      "12:37:12 [DEBUG] train episode 158: reward = -102.00, steps = 103\n",
      "12:37:21 [DEBUG] train episode 159: reward = -135.00, steps = 136\n",
      "12:37:29 [DEBUG] train episode 160: reward = -94.00, steps = 95\n",
      "12:37:42 [DEBUG] train episode 161: reward = -195.00, steps = 196\n",
      "12:37:50 [DEBUG] train episode 162: reward = -119.00, steps = 120\n",
      "12:37:59 [DEBUG] train episode 163: reward = -130.00, steps = 131\n",
      "12:38:07 [DEBUG] train episode 164: reward = -118.00, steps = 119\n",
      "12:38:18 [DEBUG] train episode 165: reward = -162.00, steps = 163\n",
      "12:38:26 [DEBUG] train episode 166: reward = -129.00, steps = 130\n",
      "12:38:34 [DEBUG] train episode 167: reward = -122.00, steps = 123\n",
      "12:38:44 [DEBUG] train episode 168: reward = -117.00, steps = 118\n",
      "12:38:53 [DEBUG] train episode 169: reward = -137.00, steps = 138\n",
      "12:39:03 [DEBUG] train episode 170: reward = -149.00, steps = 150\n",
      "12:39:17 [DEBUG] train episode 171: reward = -215.00, steps = 216\n",
      "12:39:24 [DEBUG] train episode 172: reward = -105.00, steps = 106\n",
      "12:39:34 [DEBUG] train episode 173: reward = -139.00, steps = 140\n",
      "12:39:41 [DEBUG] train episode 174: reward = -98.00, steps = 99\n",
      "12:39:48 [DEBUG] train episode 175: reward = -113.00, steps = 114\n",
      "12:39:58 [DEBUG] train episode 176: reward = -134.00, steps = 135\n",
      "12:40:07 [DEBUG] train episode 177: reward = -127.00, steps = 128\n",
      "12:40:15 [DEBUG] train episode 178: reward = -124.00, steps = 125\n",
      "12:40:32 [DEBUG] train episode 179: reward = -255.00, steps = 256\n",
      "12:40:42 [DEBUG] train episode 180: reward = -146.00, steps = 147\n",
      "12:40:49 [DEBUG] train episode 181: reward = -108.00, steps = 109\n",
      "12:41:00 [DEBUG] train episode 182: reward = -163.00, steps = 164\n",
      "12:41:11 [DEBUG] train episode 183: reward = -148.00, steps = 149\n",
      "12:41:20 [DEBUG] train episode 184: reward = -134.00, steps = 135\n",
      "12:41:27 [DEBUG] train episode 185: reward = -98.00, steps = 99\n",
      "12:41:36 [DEBUG] train episode 186: reward = -120.00, steps = 121\n",
      "12:41:43 [DEBUG] train episode 187: reward = -117.00, steps = 118\n",
      "12:41:51 [DEBUG] train episode 188: reward = -119.00, steps = 120\n",
      "12:41:58 [DEBUG] train episode 189: reward = -97.00, steps = 98\n",
      "12:42:08 [DEBUG] train episode 190: reward = -147.00, steps = 148\n",
      "12:42:18 [DEBUG] train episode 191: reward = -137.00, steps = 138\n",
      "12:42:26 [DEBUG] train episode 192: reward = -105.00, steps = 106\n",
      "12:42:32 [DEBUG] train episode 193: reward = -94.00, steps = 95\n",
      "12:42:32 [INFO] ==== test ====\n",
      "12:42:40 [DEBUG] test episode 0: reward = -112.00, steps = 113\n",
      "12:42:47 [DEBUG] test episode 1: reward = -110.00, steps = 111\n",
      "12:42:55 [DEBUG] test episode 2: reward = -120.00, steps = 121\n",
      "12:43:02 [DEBUG] test episode 3: reward = -105.00, steps = 106\n",
      "12:43:09 [DEBUG] test episode 4: reward = -105.00, steps = 106\n",
      "12:43:15 [DEBUG] test episode 5: reward = -95.00, steps = 96\n",
      "12:43:22 [DEBUG] test episode 6: reward = -118.00, steps = 119\n",
      "12:43:32 [DEBUG] test episode 7: reward = -140.00, steps = 141\n",
      "12:43:39 [DEBUG] test episode 8: reward = -114.00, steps = 115\n",
      "12:43:48 [DEBUG] test episode 9: reward = -127.00, steps = 128\n",
      "12:43:55 [DEBUG] test episode 10: reward = -107.00, steps = 108\n",
      "12:44:02 [DEBUG] test episode 11: reward = -104.00, steps = 105\n",
      "12:44:11 [DEBUG] test episode 12: reward = -141.00, steps = 142\n",
      "12:44:18 [DEBUG] test episode 13: reward = -98.00, steps = 99\n",
      "12:44:25 [DEBUG] test episode 14: reward = -115.00, steps = 116\n",
      "12:44:31 [DEBUG] test episode 15: reward = -91.00, steps = 92\n",
      "12:44:42 [DEBUG] test episode 16: reward = -156.00, steps = 157\n",
      "12:44:48 [DEBUG] test episode 17: reward = -97.00, steps = 98\n",
      "12:44:57 [DEBUG] test episode 18: reward = -131.00, steps = 132\n",
      "12:45:07 [DEBUG] test episode 19: reward = -155.00, steps = 156\n",
      "12:45:13 [DEBUG] test episode 20: reward = -97.00, steps = 98\n",
      "12:45:21 [DEBUG] test episode 21: reward = -112.00, steps = 113\n",
      "12:45:28 [DEBUG] test episode 22: reward = -115.00, steps = 116\n",
      "12:45:37 [DEBUG] test episode 23: reward = -135.00, steps = 136\n",
      "12:45:47 [DEBUG] test episode 24: reward = -143.00, steps = 144\n",
      "12:45:57 [DEBUG] test episode 25: reward = -158.00, steps = 159\n",
      "12:46:05 [DEBUG] test episode 26: reward = -111.00, steps = 112\n",
      "12:46:11 [DEBUG] test episode 27: reward = -98.00, steps = 99\n",
      "12:46:19 [DEBUG] test episode 28: reward = -121.00, steps = 122\n",
      "12:46:26 [DEBUG] test episode 29: reward = -106.00, steps = 107\n",
      "12:46:34 [DEBUG] test episode 30: reward = -118.00, steps = 119\n",
      "12:46:42 [DEBUG] test episode 31: reward = -118.00, steps = 119\n",
      "12:46:52 [DEBUG] test episode 32: reward = -168.00, steps = 169\n",
      "12:46:59 [DEBUG] test episode 33: reward = -100.00, steps = 101\n",
      "12:47:07 [DEBUG] test episode 34: reward = -117.00, steps = 118\n",
      "12:47:14 [DEBUG] test episode 35: reward = -104.00, steps = 105\n",
      "12:47:21 [DEBUG] test episode 36: reward = -110.00, steps = 111\n",
      "12:47:28 [DEBUG] test episode 37: reward = -109.00, steps = 110\n",
      "12:47:43 [DEBUG] test episode 38: reward = -225.00, steps = 226\n",
      "12:48:08 [DEBUG] test episode 39: reward = -372.00, steps = 373\n",
      "12:48:17 [DEBUG] test episode 40: reward = -144.00, steps = 145\n",
      "12:48:25 [DEBUG] test episode 41: reward = -120.00, steps = 121\n",
      "12:48:31 [DEBUG] test episode 42: reward = -95.00, steps = 96\n",
      "12:48:41 [DEBUG] test episode 43: reward = -152.00, steps = 153\n",
      "12:48:51 [DEBUG] test episode 44: reward = -142.00, steps = 143\n",
      "12:48:59 [DEBUG] test episode 45: reward = -122.00, steps = 123\n",
      "12:49:07 [DEBUG] test episode 46: reward = -117.00, steps = 118\n",
      "12:49:14 [DEBUG] test episode 47: reward = -122.00, steps = 123\n",
      "12:49:21 [DEBUG] test episode 48: reward = -102.00, steps = 103\n",
      "12:49:29 [DEBUG] test episode 49: reward = -115.00, steps = 116\n",
      "12:49:38 [DEBUG] test episode 50: reward = -131.00, steps = 132\n",
      "12:49:45 [DEBUG] test episode 51: reward = -99.00, steps = 100\n",
      "12:49:52 [DEBUG] test episode 52: reward = -114.00, steps = 115\n",
      "12:50:01 [DEBUG] test episode 53: reward = -128.00, steps = 129\n",
      "12:50:10 [DEBUG] test episode 54: reward = -149.00, steps = 150\n",
      "12:50:18 [DEBUG] test episode 55: reward = -110.00, steps = 111\n",
      "12:50:27 [DEBUG] test episode 56: reward = -135.00, steps = 136\n",
      "12:50:33 [DEBUG] test episode 57: reward = -98.00, steps = 99\n",
      "12:50:42 [DEBUG] test episode 58: reward = -137.00, steps = 138\n",
      "12:50:49 [DEBUG] test episode 59: reward = -102.00, steps = 103\n",
      "12:50:57 [DEBUG] test episode 60: reward = -116.00, steps = 117\n",
      "12:51:03 [DEBUG] test episode 61: reward = -82.00, steps = 83\n",
      "12:51:11 [DEBUG] test episode 62: reward = -124.00, steps = 125\n",
      "12:51:18 [DEBUG] test episode 63: reward = -98.00, steps = 99\n",
      "12:51:24 [DEBUG] test episode 64: reward = -92.00, steps = 93\n",
      "12:51:32 [DEBUG] test episode 65: reward = -118.00, steps = 119\n",
      "12:51:47 [DEBUG] test episode 66: reward = -228.00, steps = 229\n",
      "12:51:54 [DEBUG] test episode 67: reward = -119.00, steps = 120\n",
      "12:52:04 [DEBUG] test episode 68: reward = -146.00, steps = 147\n",
      "12:52:14 [DEBUG] test episode 69: reward = -146.00, steps = 147\n",
      "12:52:21 [DEBUG] test episode 70: reward = -117.00, steps = 118\n",
      "12:52:30 [DEBUG] test episode 71: reward = -124.00, steps = 125\n",
      "12:52:39 [DEBUG] test episode 72: reward = -143.00, steps = 144\n",
      "12:52:50 [DEBUG] test episode 73: reward = -163.00, steps = 164\n",
      "12:53:04 [DEBUG] test episode 74: reward = -218.00, steps = 219\n",
      "12:53:14 [DEBUG] test episode 75: reward = -147.00, steps = 148\n",
      "12:53:21 [DEBUG] test episode 76: reward = -110.00, steps = 111\n",
      "12:53:29 [DEBUG] test episode 77: reward = -109.00, steps = 110\n",
      "12:53:37 [DEBUG] test episode 78: reward = -123.00, steps = 124\n",
      "12:53:47 [DEBUG] test episode 79: reward = -150.00, steps = 151\n",
      "12:53:55 [DEBUG] test episode 80: reward = -122.00, steps = 123\n",
      "12:54:04 [DEBUG] test episode 81: reward = -135.00, steps = 136\n",
      "12:54:14 [DEBUG] test episode 82: reward = -156.00, steps = 157\n",
      "12:54:25 [DEBUG] test episode 83: reward = -164.00, steps = 165\n",
      "12:54:32 [DEBUG] test episode 84: reward = -96.00, steps = 97\n",
      "12:54:42 [DEBUG] test episode 85: reward = -158.00, steps = 159\n",
      "12:54:49 [DEBUG] test episode 86: reward = -106.00, steps = 107\n",
      "12:54:55 [DEBUG] test episode 87: reward = -97.00, steps = 98\n",
      "12:55:02 [DEBUG] test episode 88: reward = -98.00, steps = 99\n",
      "12:55:10 [DEBUG] test episode 89: reward = -118.00, steps = 119\n",
      "12:55:17 [DEBUG] test episode 90: reward = -117.00, steps = 118\n",
      "12:55:26 [DEBUG] test episode 91: reward = -132.00, steps = 133\n",
      "12:55:34 [DEBUG] test episode 92: reward = -125.00, steps = 126\n",
      "12:55:41 [DEBUG] test episode 93: reward = -109.00, steps = 110\n",
      "12:55:49 [DEBUG] test episode 94: reward = -107.00, steps = 108\n",
      "12:56:02 [DEBUG] test episode 95: reward = -209.00, steps = 210\n",
      "12:56:11 [DEBUG] test episode 96: reward = -139.00, steps = 140\n",
      "12:56:32 [DEBUG] test episode 97: reward = -310.00, steps = 311\n",
      "12:56:44 [DEBUG] test episode 98: reward = -184.00, steps = 185\n",
      "12:56:53 [DEBUG] test episode 99: reward = -144.00, steps = 145\n",
      "12:56:53 [INFO] average episode reward = -130.11 ± 41.24\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABgaklEQVR4nO29d5hkV3nn/z03VeycJmuyUEDZSohkhCVjkLC9a8vLLo6rBcOuvV4WjNl1XH5e22vzM14Dlg1rwPbKmGCwTRDCRiCQQKORRpqRRpqce6a7p2Olm87+ce4599xbt0J3VXd1d53P88wz3beqq27dqjrved/vGwilFAqFQqFQAIDW6RNQKBQKxepBGQWFQqFQCJRRUCgUCoVAGQWFQqFQCJRRUCgUCoXA6PQJNMvw8DDdvn17p09DoVAo1hRPP/30JKV0pNn7rxmjsH37duzbt6/Tp6FQKBRrCkLIqcXcX4WPFAqFQiFQRkGhUCgUAmUUFAqFQiFQRkGhUCgUAmUUFAqFQiFQRkGhUCgUAmUUFAqFQiFQRkGh6DBfOnAesyWn06ehUABQRkGh6CgT8xX8p//7DP7hwPlOn4pCAUAZBYWioxQqLgCgaLsdPpPVhe36cDy/06fRlSijoFB0kLLrsf8dtQDKvPezB/Dgp1Rbm06wZnofKRTrEW4MKoFxUAC+T/EvL01gKG91+lS6EuUpKBQdpOwoTyHOy5fmMVtysFBefEitZHs4NVVYhrPqDKenivjO0ckVDaW1ZBQIIf+aEHKIEOITQm6J3fZ+QshRQshLhJB7pOM3E0KeD277MCGEtHIOCsVaJjQKylPgPHXiMoBQb1kMn/jOCbz5Tx4HpbTdp9URPrPvDN7+ie8v6VoslVY9hYMAfgzAt+SDhJCrATwA4BoA9wL4CCFED27+KIAHAewJ/t3b4jkoFGuWMHyUvBOcLtj4yT97Ameniyt5Wh3lqZPTAICC7cH3F7e4n50uYr7swvHaaxQKFRffOz7V8uM8cWwKJbt6A/D/P/oy/vxbx6uOP/riRdxyxQD6sysXSmvJKFBKX6SUvpRw0/0AHqaUViilJwAcBXArIWQjgF5K6ROUmfJPAXhrK+egUKxlKm59T+HA2Rl878RlPH1qeiVPq2NQSvHUycvi98Iis7KmFmwA7ddo/m7fGfzkQ0/i3ExpyY8xMV/Bv/mLJ/HwU6erbvvMU2fwtUPjkWNnp4s4PD6Pu68aW/JzLoXl0hQ2Azgj/X42OLY5+Dl+PBFCyIOEkH2EkH0TExPLcqIKRSdppClcnCsDAC7NVVbsnDrJ2ekSLsyWcdXGXgDAQhNhk5fG58UufrrIjEK7NZrx4Prvb8E4X5wrg1Lg5YsLkeNF28X52XJVAeM3XrwEALj76lVmFAghjxJCDib8u7/enyUco3WOJ0IpfYhSegul9JaRkaanySk6xFzZwV89eWrdxHNXgkbZRxdmmVGYWOgOo/DyxXkAwB07hwA0pyv8j396Ae/93HMAgKnC8ngKlwuBUTi9dKMwMc8e48Rk1CicnGShwbhRePTFi9g5ksOO4dySn3MpNExJpZTevYTHPQtgq/T7FgDng+NbEo4r1gFfOziO//b3B3HnriHsHMl3+nTaxuNHJkFB8eo97d+YcE+h0tBTKLf9uVcjc2W2MG4ZyAAA5htkIFFKcfDcLIq2B0opLguj0Lqn8LHHjuHLz1/Al959Fy4X2HntPz2z5MfjRuH4RDQ76nhgJGYko1B2PDx5fAo/c+f2JT/fUlmu8NGXADxACEkRQnaACcrfp5ReADBPCLk9yDp6O4AvLtM5KFYYvqubW0Iq4Wrmj7/xMv740SPL8tjcUygrTwEAMFdin51N/WkAQKHi4eC5WXzqiZOJ9x+fK2O66KDi+pguOpgpsoW1HdlcxycWcPDcLDyfCk/hhfOzDR97fLaMP/r6y1UiOX8PL81XMF8ODQA3Erbri8ceny3D8Sj2jvW0/DoWS6spqT9KCDkL4A4A/0QI+RoAUEoPAfgMgBcAfBXAuyil/Eq+E8BfgInPxwB8pZVzUKweCkFWhfyBXw8UbU+8tnbDjUEtT2GcG4X5bjEK7LOzsY95CgsVBw8/dRq//Q8vJIYlD52bEz+/ND4vfm6Hp1BxffgUmFqo4HLBRk/KgONRHDo/W/fvvvz8BXz4G0dwdjoqSsvv4YnJQuLPPITEPcQNfemWX8diaTX76AuU0i2U0hSldIxSeo902wcppbsopVdSSr8iHd9HKb02uO3dVAWg1w0lYRTWl6dQsj2Ulqk3kRCaa3gK4zx81CVGYb7iImPqGAhSMBcqHqYLDlyfophgmF+4EBqFF6Wf2+EpcEN9ab6CqYKN1+xl4cP9p2bq/t1k4BHMxTZHEwsVmDqTVeUQ0vGJUGPgRoG/7xt615hRUChkePrgevMUSo6H0jIVl4nwUcLjlx0PM0UHGVPHTNFZ1lYYRy7ON5Xps9zMlRz0Zgzk00zuXCg7IqNoJqG9+KHzs0ibbBmTjUI7PAU7qCI+N1PCfNnF3rEebBnI4NmzM3X/jqfFxo3C5HwF12zqAyHA8cA7oJTi+GRBiMlxT2FUGQXFWma9egpF20vcpbYDITQnLGJ8Ybh2M0vP5IvNcvBjH/0uHnrs2LI9frPMlR30pE3kUqzWtWB7QjyeKVa//kPn53DnrmEAwIvjklFYhBE/cGZGhOlkuBF+OQhLDeYt7B3rwbFLC1X3lZkK9If492BioYLN/RlsGcjg+MQCvn/iMp45M4P5sosbt/YDAGaL3ChUkDF19KZXvj2dMgqKtsHj7qtZaC47Hr7x4sVF/U3J8RKrUNt1PvL/MlxkvnZzH4DlCyG5no/5souTU52vmp4ruehNG0gZOixdw3zZFZ4CXzA5syUHZ6dLuPmKAfSkjUj+f8X1UbI9PHJoHJRSXJwr4y1/8ngkfg8wHeJffey7+P2vHa46Fx4+OhwYhaGchd2jeRyfLMCrU2k9wT2FUrWnMNKTws7hPP758CX8xJ89gR/7yHcBADdu6xevCWDho7HeFDrRBUgZBUXbKK2B8NFXD47j5z+5r+m2EZ5PYbs+3OD/ZpgtOXjDH34T3z062fC+cvFaXF7jnsJ1W5hRWC6xmXspSbvllWau7KA3YwIAcikdCxUH00E6aDyP/4XzzDO4ZlMvxnrTkfen7Hh45IVxPPjpp7Hv1DS+dmgcz5+bxXNS6Mf1fPzXzx6A41EcvjCPOPy6cA9kMGdh10gOtuvX/fxMLVR7CmXHw1zZxXDewq6RPIq2hx+5biPuv2ETBrIm7gi8Hf4aL82VMdaB0BGgWmd3HV989hxu2zG0LFkNhcrSw0e+T6Fpy78r4l86fq6NkLWEkuPBMhrvo168MIdjEwV85JvHcOfu4br3lStvbc9HytDF79xTeOXmfgDLZxS4YTo/u/QWDu1ivuxi+xCLr+fTBi7NVURsP64p8Cygazb1Yaw3haOXFkAIQClb0LmRePSFizgWiLnThTAE9eknT+G5s7PYM5rHsYkFeD6FLn0G+d+fDLyLwZwlhOJjEwu4IjhP1/Nh6OxzQSlNFJr5sZGeFP7VzVtxzaZe/OiNm8Vnnnsesqdw07aBJV3DVlGeQhdRdjz80sPP4nP7zza+8xIoBovLYlsen5gs4Nrf/Bq+f+Jy4zu3CNcGmt31yxPRmg0h8UXk8aOTYjGaXKjg44+fqPIG5KyjeGuG8dkyelIGrhjKAgAuzS/PTr4cXIuLc+VFN6BrBxdmS/g/32HXZq7koCeIo+csA2ektM6ZWPjohQtzGOlJYaQnhdEetskZzqcAsNAPf6+/dmgcTxzjbTDYY1RcDx977Bhu3zmIf//qnai4Ps5cju7+uabALwnzFFhR5tFAV/jy8xdw3W89gm+9zNrwFG1PvI/y5mgyCCmN9KSwoS+NH795S2QTpGsEPSkDsyUnCHdVOuYpKKPQRVQadORslWKQvTJfWVz46DP7zqBoe+KLtZzwEJftNbfAl+3wWjU7MvPEVAGGRmDqBH/15CkAwD89dwG/848viFRD8fiSIYiLo+OzZYz1pWHqGgZz1rJ7Co5HMVlY/tTX2aITMY6f338Ov/UP7NrI4aOetBFZqGdKUaH5hfNzuGYTE+FHe5kx2BR4wGUnTA44OVUUehfXJz6//xwuzlXwrtfvxu4xttAfiQnI8veEEGAga6E/a2E4b+HopQX83b4zeNff7EfR9kSjPDkZQNYU+HvHjVYSvRkTcyVWgGe7vjIKiuWH70rdZRrYUVxC9pHnU3xh/zkArCPocsPPsVaxWNX9nfC1NJuBdHKygCuGsvihazbgS8+yLi58lxt/XtkQyItQyfbw/LlZbO5nRVyjPSlMzFcwPltu+/sni9zLrSucuVzEzf/j63jgoSdxJOhzxFt4nJwswvEoetNcUzAiabJzsTYQRy4tCKMwFngKIz1p6BphQrPjgeu0hkYw2pPCdNGB71N87LFjuG5LH+7aPYw9o8wo8L5LHPn96M+YIrS0cySP587O4oNffhFXbWDPzz1PufJc/h5wozDSU9so9GVMzJacjtYoAMoodBV8QXKXKURQFEJz80bhu8cmRabFc2dnl72ZHg9xVZpcWOWQUbMFUScni9gxnMOe0TymCjZczxfxZTv2vGXHE3n28uP/4SMv4dxMCf/hNTsBsMXksZcncPvvfqPt4T958buwzEbh+XOzcH2KA2dn8POfZDOYeVbVkUtsUe7NsPBRPhVKnpauRcJHRy4yDeCaTUyE57vqoZyFtKEFnoKLwayFazb14tYdg9g8kMFM0cbkQgWnpor4sRs3gxCCnrSJTX1pERLi2K4PIzAEg7lwnsHu0TwOj89jpujgV3/4FQAgJqNxkTltahGPmWsKQ7nGRoEnGIz11r7vcqKMQhfBPYXlGu1XXEKbiy88cw49aQPveO0uzJYcnFrmtEge4mpWU5CNQjOegu9TnJwqYPtQDv1BGGSu7Ipdbvx5y64vqnd5KOnguVl8/Dsn8G9v3yaE6l0jeVDKwhjtTh2VjdGFFuYFNMORi0wM/rlX7cDpy0WUHU8YBd6mgnsKslG4YigbMQpcZL46aLHNF9DBvIWUqaPi+ihWPGQsHX/5s7fif/+bmzCQtXC5YONi0AZ7U+CFAcDusZ4ET8ET95GNAtcV7tw1hFt3DAb3Ze8d1w62D+VEHyeAeQr9WbNuokK1UVCegmKZEZ5Cm6dSscf0xRdjMXUKJyYLuH5Lv/hyLXcISYSPmjUKzuKMwvhcGRXXx/bhHPqybHGbLTkiqyRelVx2PPQFxoPf9sihcRAA//WeV4j7ve/eV+B7v/YGjORTuNzmIjY5pHVhmbuxHp1YwOb+jFhYx2fLQkA/EtQZcKGZGwWNANsGs5gpOTh4bhYPPPQEPr//HPIpA9sGmQjPheaop+Aha+kY6UlhMGdhIGthpugkVgvvHc3j6KUFkQXk+RSOR8Xjy0bhhq39IAT4jz+4B1aQdRT3FHYM5yKbo7PTxYbhoNAoVILzU56CYpkRmoLffk+Bh2UGsiZs12+6JUOx4iGfMrB3rAcpQ8NzZ+s3G2sVvsjX8hS++dIlvO0vnhRfctkQlJzGxo5nHu0YzonFfqZoi/CRbIwopRGjwD2FZ87M4MoNveI4AGQsHQM5C4M5S8wMaBdRT2F5jcKRi/PYM5rHxqAL6vmZkhgg9NJFHj4KNQUgFHhnizYeeeEinjx+Gd8/eRlXbewRGTxbBzP4L2/cize9cmPoKTgeMlbobQxkTUwXbVycrw7P7BnLo+L6OB0I2/zzsXWQewrhfW++YgD7/9sbcceuIWgagaERcf+pgo2etIGhvCU2R75P8cyZGVy/pb/utenLMqNwfqaEwZwVSU9eSZRR6CKW01PgYRbu8jarKyxUXGRTOkxdwzWberHv5OW61aJJPHXyMn7kw99uKuYfegrJ9/3usSl85+iUGAm52PDRiSlmFLYP59CXYbtL5ilUh60cj8KnQH+WGwU2k/jZ0zO4KahwjTOYs0Qb528fmaiq0F0K3FCN9qTaLjR/99gkTgXXxPNZn5/do3lsCrqgHh6fF8/PvSkePuIew0DOQn+wYB6bWMCWgQw++KPX4r33hp4UIQT/8Q17sKk/g5ShoeKyJoZZM1xYB3IWiraH05eLICSaCXRjUBPwZDDBjb9PWwaYpzCcj85IHpA8B8vQxCZiYqGC4XwKvWkT82Un6G20gJmig5u316876MuYqLg+/vnwJdH2ohMoo9BF8EVzKULzLz38DD709Zdr3s5nKSzWKBRtF7lgN/eGq8Zw4Ows3vqn36k5C9f3Kf76e6ci1a1Pn5rGofNzQsyr/3z1PQWeJfLoC2wUYqR4rQmjcHKygJShYWNvWuz0Z0tOoqbAPbcwfOTjyKUFzFfcmoVLzCgwT+E//d9n8N7PHmh4TjJJQj7/XOwYzuHCXHs1hV/52wP48DeOAmCZR7brY89ojyie5OFCXhQGhEIz9xQGsxb6MyYKtofDF+Zw5VgP3nbbFfiB7YOJz5kydZQdH4WKJ3ooARDazUvj8xjKpWDq4fK3ZzSPDb1pkRbNNw39WRMf/qkb8VO3bqv5Gk1dCz2FhQqG8xZ60iYcj6Ls+Nh3kk1ru/mK+kaBe0iX5it40ys31r3vcqKMQhfBd2RLEZr3nZyOtCmOwxfbDcIoOE2JuQXbE1/+X3zdLnz4p27Eixfm8OknTiXe//snL+MDXziIrzx/QRwLB6s0fj6eIVXr3Lhh+cbhi6A02q6ZGwXPp7jr9/4Zn326Ogvo6KUFXDGUhaaRqFFICB/xxbhfCM0engnGPd5Yw1MYCoxCyfYwXXTw1MnpSHfQejx/dhav/M1Hqlo0yEZhfLa9BWwLFVc8H8/u2T2WR9rUMZy3cODMDADgFUFqJ1AtNPdnTeFNHZsoYNdo/al+wlNICB8BrMFdPLOHEILX7B3G40cnI/qYpWu47/pNEVE6jmVosAPve3LBxlAuJQzbfNnBvlPTGMxZ2NlgrCb/vJg6WfG5zDLKKHQRwlNYQvhooeLWNSZFET5iX7YDZ2ZwzW98Fc/X0Qgcj7UiyFlsN0cIwX3Xb8L24Vykx7wM38ldlAbZz4hh7eEC/sSxKfzjc9WTXhsJzRPzFWgEODVVxLGJBeEpGBoRuknJ8XB2uoTvxHobeT7FvlPTuHEr2xHyL/l0wRGeU8RTCArjhKbgeth/ehoDWbPmXN7BXApzZTeysH/6yWQDGufRFy9ioeLidCx7iVc07xjOwfFo2xrvUUpRcjyR5sqLw3YHi/rGvozIpOL9nSxdQyrI0OFGYTBniV00AOxuMOo1HXgKxVj4iBvf87PJfYVes3cE82UXB87OCE8hZTaO61txT6GHeQoAa3Xx9Klp3LRtoGFzO/45eM2ekYietNIoo9BF8IVwsUIzpbQJoxCEj4KwwNcOXYTjUTF/NvFvgv5DWSn1EAB2DudEv/k43zrCjMLEQhj75lWqsk7w8cdP4I8Swl2lJsJHr7tyFADw9RcuoWSzoS8ZSxd/y/8/PB5NYXzh/Bzmyy7u3M2GzluGhqylRxZwuU6hKnzk+Hjm9AxurLOADAaxbe61XTGUxd8/cy6xrXQcrpPEp8hxY3p9EMduNFmsWRyPwvOp8D6OXlrAWG9KeAJ85CYQGoXejCFeez6iKUgpoaP1d9zMU2BtLjJWuKjLGURJNQB37R6GRoDHXp4U35VUE72uZE1hruyiN22KltcnJ4s4MVnALQ30BPmc3nx950JHgDIKXYXczmAxFG2Ppei5tf9OeApBaiBfgOL9amT4UJ58Krob2zmSx6mpQlXl7uRCBQeD8YuXJE9hOqFauOx4VdXDLBzE21xUGwXX83G5aOOVm/uwZSCDl8bnghCEjqxkFPh1PHZpIWIonzjOPIc7dg6JY30ZU2S0sHOsLobjoZGS4+H4ZAGv2FB7Lu9gsDjyDqHvu/cVqLg+fucfXwSlFEcuzifqBo7HDA5Q3a6j4vowdYLrtvRBI8CBNmWAiUwvz8dkoYKjl+aFlwCEIzfTpobdo+w1c4MBQGhNXFPg7GrgKaQMDZWg3XnWkjWF8DFGeqo9hf6shWs39+GpE5cXZRRMnWUfuZ4Pz6dIm7rwFL75MtOmGukJAAuhff4X78Rbb9jc8L7LiTIKXcRSPQXeaiBpIeUITSHwFOIZJUlwcTprVXsKjkerxObHj7BFdzBnRdoJiPCR5CmUHK8qw4jP3AWSh7BcLtiglFUPD+dTmCrYbLdp6shahggfCbHa80UKKsAyl3aN5CL5730ZM9K/J+IpBEaLL4TnZkrwfFq3gy3f7XJP4bV7R/CLr9uFz+0/izf/yeN444e+hW++VN1D6uC5WbFIxzvElh0PaYO9xr1jPThwZgaUUjwb/L9U5HDe+Zkyjl5awJ7R0OBxT2GsNy1ec4+0+I/2pmBoBNuGssJwDuejXkMSaVPHXNmF61OhVwGI/F2tauHRnhRmS47YUDTTFZd7CrIh6Qs0hX85PAFL1/DKYCZGI5oJMy03yih0EUvVFHgRTjPho9FYb5f6ngI7n3w8fDTCwgPyHFsA+PaRSQxkTdy1ezjSHG46QWiWu1XKxzhJBu6S1LRsOG9hasFGOfAUMqYumunJGUk8t97xfDx14rKYAsbpy5iRgjDbZXMTvnt0UrwfGUtHytCE8YhfQ5mhfOgp9KQN5FIG3v2Du3HVxl6hFZycqg69cc+NXYeop1B2fBE7v25LH547O4MvPz+Ot/7pd/BMIAQvBTlba/+paRRsLyISc09htCeF0Z4UCEFk0thwPoVvv+/1+KGrx9AfpPc28hIAtijzjUJG0gQsQxOftbEETwFgGU8F2w01hSZqBUxdg+2FrbpThiY8hXMzJVy7uRfpJrSJ1YIyCl1E6CkszijwIpx6xoQvuPm0IYRjoLqzpUzoKVSHjwCIttOcS/NlXDGUw4a+NC7NV0ApBaU0UWguO15V3YK8GCYJzXLP+6FcClOFiqiKzVi6eI3yYsdbMzxzegYF28Mdu4Yij9mXMSFvtm3Xx/7TM/g3f/E9/PNhFlpIG8wo8BYf9ebyck9hqmBjY7C7Thk6PvfOO/Dkr70Bpk4SheKnTk6LQqz4LOaK1H/pui39mC46+INgElkro1Vl48m1oD2SUdgkmv2xTrAj+VQkfAQww8H6ExnQCBpmHgHMU+Cf8fhni3sctVpI5FIGChU3ssA3wtKZhiEylgw98jpuqZE6u1pRQ3a6iNBTWFz4iC8MdT2FigtC2ALXk2Y55WO9qaqRhDLcKORinsJgUKwUF5tt14dlaBjtScF2fcyVXegaERqJ7BmUbA+uTyMDUOTFPMkocO9jtCeFwcBTKNoe0iZbtPliyq8jIaHY/InHT6AnbeDVe6KeQn82ushVXB+zgaHkU8DSpoa0qYtwWb2eN/0ZUwySke/HQ3DD+VRii+2D52Zx+84hXJq7UFWEV3F9sfjxqlueFeS1UP0uG2VeFLY7YhR4Z1PmGf3WfddgY43UT00j+J8/dh1uaiI2Ly/kmZhRGMxZODtdqhk+ygedWfnngxvLeljBZyP0LjSkTQ2GRuD6tGPDcpaK8hS6iLBOYXGeAh+a00hTyJg6NI3t6nh/m3rhI744xY0CwNIjT8TCR47HFi++iEzMVyKTtGQNgXsF8uIfCR8lGYWFMHw0lLPg+hSX5srMUzCl7KNgsds1ksdL4/M4cnEeXz00jp+5c7sIG3Dk1ELR0tmOzv5NmzrSpi4quUfq9Nw3dE085sYE7WG0J1XlKdiuj/G5MrYNZpEPdsIyrFMrWzyv3NATiaMnfVZ+6qEn8dffa5wGW4p4bj4GsiaGpAyg0Z40rhzrEQv9D79yI26oU8n7Ez+wNWJUaiGHauJ6VX/WgkaAoRrXOGcZIp0VACy9+ZRUoSmYGgghIo22GZF5NaGMQhfBxdXFCs1xTeGZ09P4lyD0wSnYnvgC3rFrCPfdsAn9WbNqhKIM33nnrOov3s7hfFU6q+NRFmYIjMKl+XLE6JSdahG37Hg4OVnAZ58+K7KdgKhRoJTNX56ct5FPGchYumiBcG6mFAjN1eGjm7b14/TlIv7tx7+HjKnjZ1+1o+p1cHFTI2yXb3u+2EHzx+OeCMCK0xqJmzyElNRgbaSn2lMYny2DUmDLQAbZlF7lKZTd0ChYhoYfeeVGvP7KEQCoajliuz6eOD6VWH9y8NwsvnZoPHzc4HVynWD3aD4iouoawdf+82tw3/Wb6r7exSJ7CvHw0Ug+hQ296cjYTRleAc11qlQTnoKpB0Kzw0NO7DF60ga2D2XrzlBYjSij0EWEQ3YWKzTz8BH7uz977Dg++OUXI/cp2a74Av72/dfiffe+An0ZM7Jon5gs4OOPnxC/891YvE4BYGLzxblKRAdwPJY6OSp7ClJ+vhwe415N2fXxt/vO4D1/dwCzwblYQcUr5/e++hLu+9+P49J8WXyBuaDreJQJzZYRGoXgeX7+rp14x2t3YTCXwrt/cHckD57Dd4u9GTNIlfQjO2ggDB8B9fUEDt9tb+irDrUkGQVeJ7F5IIOcleQp+JEwyYd+8gZ84EeuBlCtP3HdJa5LAMBffPs4fv2LB8Xv3CPiGtFuKfNoOalnFH757j34yL+9uebfcq+Ve6DN1inYri+m+fG/ecMrxvATP7B1cSe/ClCaQhfBdzLOYj2FYAFwXL7QelV9gAqxnHAA6MtYmAtmzhJC8IVnzuHD3ziCt922DWlTF6mR2YTMDL7AzpYc4YHYnh8IkmzhjC9+3OiVJS+g4nhihgKPk/NOrpyz00UcHp/H+ZkSrgxqBORhKBlTR8rQhdHhr31Tfxq/+sOvEINWkuChnt40m9wlewocFj5iC0m9zCOO8BT6qu870pPGVKES0VLOBnOOtw5khcfj+RS/99XD+LlX7UDZ8SJ1AADEcJm4/sRDU0lGoWB7kU0AN347R3J49sxMRGReTuqFj7YOZrE1aIedBDcKU8IoNJd95HhU8hTYdf/1t1y9uBNfJSij0EUs3VOITg2z3erdbrxQCGAiq+2x+2YtQyzOXLwtVJh3oSW48jx1cKHsAkGKt+P5sHQNvRkDlqFhYr4SaWrGv5Syd1GWBrjzmoKBrBXRR+RqVO4pyF0xsxYzCkXbFa0bADSVZtgvPAUDjkthxwyqrhGYuiYWn2ambfE2zht6kz0FStmi9mePHcd9N2zC2ZkSNMJqSPiIy+MTC3joW8exbTDLhOZYmMQIGtTFPQU+OjPubQDsuldcX2gUsvYCoCk9oB3IryX+mWwEL6TkTQebrVOIZh+t7QCMMgpdxFLHccazj2y3erdbsN2qegO+IM4U2W6f1yUUbReDOSuiQ8ThLQ7kgT22yzwFQghGgiwbnl0ymLNESIj3FAKYIeRFZ7zNdF/GFF969rrC68G1BLk1csbUkTJ1+BRi9q+pk4hBqgX3FPoyJuZKLNVRLrJLBwsI9xSambY1mGOPmVTkxkXqZ07P4BPfOYGFigPXp9jQy9I+c5aBi3NlofXMFG1RvCZjaOx84hsI7ikkpapyz2+u5LD+Q8H7fc81Yzh6aWHFBFf5tcSzjxrBq6gvF2wYGqmpPchYOgmK15qvbVjNKKPQRYSewiIrmvmwEMqER77752EhgHkK8awZuUvopv6M2MGXJOMQb3HB4eKkHKZwPArTYM830pPCxEIFvRkzKOLShbgczXrxxOJ0XPIUxqWCMp7VVHF9YRTMIMtntuQgYxli0S7ZbKffbDGSHD6qOL7IPuJxaP44qUVoCm++bhMISKRtA4dP6+KJAPtPz2AwZ2HzAPMqsikWtuMx8+miEyle43BPIZ6Syo1Cwa42Cvx9nS05GO1Ni/fhiqEcPvSTNzR8Xe2iFU+Bh48uF+ym9AQg1BTk7KO1jDIKXcRSh+zIA8h5Z1Ma7Jr5olaQhGYOH0fJ48x8Jxm2W3Brewop9rfySEMn8BQAFns/NVUUYxZTwQhG+fH5a+bho8mFCghhWSGypmC7Pq7f0o87dg3hLVImzFDeYkbB1MSOsxgUxWWaNAq8TqE3zQwMD731ZUwxyhEI49BjTWgKV23sxVUbexNv44b5X15iRuHopQUMZE3R5C9nGSjarnhPpot2pHiNwzWFeErqRDC1bCHJUwgMBW9tshiPqp3wnTqvm1kMedkoNPkei+yjRRS8rWbW9tkrFgX3FBYtNJejGUB8QZVDSMWKV5VF1C8mj7FdKfcU+CIdH4Iiw6duyYuPHWgKAPMUzs+WMLnABqKzdsmhB8KpSOEjgC2KKVOL1C84Houp/+c37o20rB4OYvdZyxDGr2R7okleM/SkTZaOGgxt50JzxtSxcyQnHoc/fqvD2sN03YpY2KeLDjb3S56C7YlK85miEzHuHC5Sx1NSeSPCeP8kIOop8N870d6BGzheN7MYuKewUHEjRrselqHB9an4/K318JEyCl3EUj2FhXI0hMMXVL4jvzRfxnTRrsqcqfIUYnn+RdtNLFwDQk0hGj4KPYXXBr3vnzg2hX7hKVQbq7Lji/ARwGLMlq5HPQXpcWV4Wmra0oVHw8NHzXoKukbwxw/ciLfddgVrh+BwIVbDf73nSvz6m1mGStpoj1FIm7oIvf3QNWPga+KWIHyUs5iXNLnAjMJUwYbt+VW7W+EpxDYQfL6x7VXP4Y57CovxqNoJX5QXGzoCon24mg0D8c8O3zytdaF5bZ+9YlHI4zgX0/1yruyKxcWRawCCRfiLz5yHT1msW6Zf0hQARLKPALbg52qFj6yo0Oz5bJ4x/wLefdUY9o7l4VOWYpo2dbFIlWSh2fFQdELDkrV0EQPmOC6NjIPkcKOQDYrX2Lm7i/IUAOAt12/CtqEsUqYu9JiMqeO6Lf14zV5WJJZPGzB1UjULeClwb+G2HUPYO8ZSbLmmwI0wb6lxMRiAU+UpBG+4FxeapZblsrfg+VR8HiJGYQkLc6sIT2EJz502NfFZbzYMxO/HjYIKHynWDHLIZDEZSPNlR8y3td0wfMR3/J/bfxbXb+2vSjnMWjpMnYhMl3jxV9GuHT7SNML60MQyn7jQrGkEv/i63QCYcJw2tZpCs5wCmgmqh23PF2MnHc+HleDy89TPTNAQD2CawmI8BRneDiEprPL2O67AJ3/uVhG2aQVuFK7d3CsG0vMB9Lx6/FxQu3Ap2PmnYwuZLjwFdo3myw48n2JyoSLaa8gepHzNZU2ho56CuXjJlBAiDGezYSC+UVkItDdlFBRrAkpZ2Ie7tvFYcS14VgUvmJI1hZLj4dD5WRwen8e/uql6MAghBH0ZSwof8ewj9v9CHaEZYK58vEZCjvO++bqNeOPVY7hr9zAb1i48hWg31JLtiUUulzLENRB1F0GldBy+a89YuvBoihVvyYsdz2dP2kEP51NVbbeXymhPGoQwQfpHb9yM1105IsJH2ZinwD8GcSNFCIGhEXi+j4WKix/44KP42GPH4NOwtbkc2pN1nNAoVGsVKwFflLM1NhyN4CGkZsNA/H4LZReWoXV8HkKrqOyjLoF7CfmUgcuuDcdr7gvLv/gDubDtQxg+8vDtIxPQCCJZOzJ9GUN0SuXjN4u2F0xBq+0pAExsXohVU8uxf0PX8OdvvwUA8JWD40IziTdiKzoerhjM4uRUMShE08Q1SZu6KIqLszXYXQ/nUiK+PFOyUXI8pJcQmuAD5cuOv+ismMXwplduxHA+haxl4NYdg7h1x63iNu4pxKvBk+LnukbgehRzJZa2+mePHQPAmhV+5+hU1ChUqj2F8hI9qlbhWUNL0RQASJ7C4jSFhYq75r0EQHkKXQNfMPkuqFmxme/U+RjIsuMJL6PssHz3nrRZcxpWf9bCTMmOaBFF20PFZaMLawnNAIuzx/su1UpvZBlFUU1B1whmSw4oBbYHWUU8fASETfF4o704r7tyBI/859dEJn9NF+wlL3apQMtYrCaxWO69dkPNFguyZybvhJOMlKlrQftxdu25vrNzmIUJCxFPITQKc1L4qJnW0+2Gv7+ZJYSPgNBwLqZOAWDXZ61nHgHKKHQNfMHki3AzaakHzsyIMAMfGC9/+UuOh/lKdSWzTH/QFE/+u7LjhbMU6oSPetJm2HeJawoJYR6ALWqypmAZGrKmLhrmbR9iRoELzUAYPpLrH2QIIUKoTRk687IKTtC2Y2nhI9vzRZuPTiC/V1dIPYCSzod5Cn5Vy/QdQfhovmH4qDNCswgftewpNPf3VvCZXCivD09BhY9WAYfH55CzjLqNulqlLDwF9kFv5ClQSvGTDz0hag24pyDvDks2W9zrGYW+rInD4/ORRaNoe8JI1Pvi9qQM0eFTaAo1vnRMaA41hYypw9Q1YRR4/UHGMsSXnbcStz1fCNj1GMxZuBxMY1uq0EwpEyQ7EVYBonH2HcM5HLnE2pMnhY9MnQ2J4a3WedX3zuBaJnkKI8GMY6BzdQqEEKQMrWWjsGhNQYWPAELIHxBCDhNCniOEfIEQ0i/d9n5CyFFCyEuEkHuk4zcTQp4PbvswWeuqTBt4z98dwP965KVlfY64p+B6FO/97AG849NPi9m+MoVgxjFvB8GF5kKk2ZyHhYoragqSGMhamC7akfTFou2Jx6lnUHrSCdlHNcJHfASjK6V8pgxN9MXvz5rYO5bHrpFcxFOglEaK4uoxkLMwVbATi72agS+8ZcdHxurM4iF7ZnKhXm1PIQwf/ac37MFv3XeN+CzI2Ufc6G/sS3e8TgFgTfiW2oAv34KmsNZrFIDWw0dfB3AtpfQ6AC8DeD8AEEKuBvAAgGsA3AvgI4QQ/un4KIAHAewJ/t3b4jmseeSd83JRjmkKju/jsZcn8NVD47j3j78lul9y4mM0ec6+LC6WHR8LFa+uLjCYs1C0vcjcg5LjhvOZ62kKKVlobqApBF/Gsuuj5PjIWqwdNe/xk7UMfO2XX4NfePVOYQDsQNegtPbjRl5L1sT5IJy2lLCIbHiWU2iuR9xT4CQtgIbGNAXupV29qRc/fed2YVjmEzwF2Sh0KiUVAL78S6/GL7x655L+lic/NFu8ZoniNafp1hirmZaMAqX0EUop/2Q8CWBL8PP9AB6mlFYopScAHAVwKyFkI4BeSukTlFVPfQrAW1s5h/WA71ORM98Onj51GTf89iO4MFsSx7inwI2C57MU1a2DGRRtD6cvR70FLvC+5fpN2DmcE20SIuEjx8NC2UFPnYWd1zfwvHgg8BQCzyFp6honn2aDbeS4dk1NwQxDQjxskTZ1ySjoIlWQf9krrt9QwI68lpyF8zPMeC4tJXXp3TvbhTy7YutgVhRqJXkKhk7g+r7wFMygc6qmEeQsPfJZKAijkGFV5M7i2oGsJhZdpxAYVMejKnwU4+cAfCX4eTOAM9JtZ4Njm4Of48cTIYQ8SAjZRwjZNzEx0cZTXV14lMJfRIVxI77x4iXMFB08cWxKHKvyFILxgbxAKz7Inmcd/eubt+Cf3/M6MUFsoRIVmuv1LwLCNs9cG8hautAigOT5zBw+77hQ8UT4qFaYh2e5ME/BRcZiRoHvZuVFT/YUGmkVMkM5S6S7LjX7KDzfziyWhq6JazWQtUQX10SjEAye5111Dckg56XQHhDWhvDCton5Cijt3OtsBV5N37SmkNDYcC3T8BUQQh4lhBxM+He/dJ8PAHAB/DU/lPBQtM7xRCilD1FKb6GU3jIyMtLoVNcsvg8ssh1RXfafno78D4QtLmRNoex6ok9OfJA99xR4Yzre9iAuNC9UXNHRNAnuKfDpX0N5trDyWP9AjVRWAMIDmSs7UkVzbU2Bv05ecVxrLKMlpaSGxqaxtCXPWFhKnYK8yHQqrAKEukJ/1hTXP17RDATho4iXFt4nlzKwIOlL3PPjMx4uzi3do+o0i61TkN/X9WAUGmYfUUrvrnc7IeSnAbwZwBto2FDnLAB5OOkWAOeD41sSjnc1XhvDR67n48AZNlR9/6kZcZx7AnyRZwVkrKWzfDtnLvAU+G6dLwhV2Ud1ZiIA4ULKjcJwPoX5sis0hv6EmQCcHqkpXiOhmbv6LGzhYyivR6q2ZaMgso9cr+HjygxKBixphGgjIkahg2GVbErHVIEZZH79k2Lhhk7gSXUKcuiuJ2VUtblIm5qoV+EJCmsxfJRfdEqqbBTW3uuN02r20b0A3gfgPkqpHJT+EoAHCCEpQsgOMEH5+5TSCwDmCSG3B1lHbwfwxVbOYT3gUdp024lGHB6fR8nxsHM4h8Pjc2IRj3sK/Hhvhv0e73jJC5W4J8EXtIIkiE8VbFCKhtlHQNhWYTifQsn2MLVgIxeEeGrBH3e+7MJ2qxcmGTmzh2e9yIVT8o6Vv5aK6zcUsGUG5WlsS6xo5nSiqIuTs1irj7SpNfAUWPgoyXDmUkZUUwiaG/Jw1Phs93gK5jrzFFp9Bf8bQA+ArxNCniWEfAwAKKWHAHwGwAsAvgrgXZRSvpq8E8BfgInPxxDqEF2L71N4bdIUnglCRj971w74FDhwdgZA6AkIo2DzRZ99iavDR409hcmFSuQxk+A70XPCU+DhIzsSjkmCP/dCxWmsKUi7/2JQp1BrLKNc0Ww3CEvJyEZhKbFyaxVoCgDzmgayJggh6M9aMDSS2IjP0DW4HhVN8WSDzDPD/uQbR/CX3znBQnaWLowCDx+tRU2Ba2RL0hTW+NQ1oMXiNUrp7jq3fRDABxOO7wNwbSvPu97waPvCR8+cnsFwPoW3XLcR//3vD+KZ0zO4c9ew8BR4qCeuGcSrVufLLkydiB0tXxB4iqihEdE/p16tgalr6E0bmAuahfWkTRRtF5cLNoYaGAX+uPNlV3hStesUgt2/44sFSnPDRUzescq9j5aqKbQqNHdUU0gZoihxz1i+ZtGkrvHso0Bo1sLzz6cMTBdtfOyxY9gxwrLTZE/hvGjJvfYWycXWKay38JGqaF4FeG30FJ49M4Mbt/WjP2th10gOz5yeASB5Crzbp82NQm1PoSdtijTOuKfQlzGFp1DPKABshz1XdpGzdGRM1o5icqEiWjzXokcKH3Gj1IzQzIrDotlG8k5YFprthEZ7NV9HtrXwkaV3PiUVAN75ul1CGP73r96Jn7lze+L9TJ2g4viJIn8+beBiMFvhzOUSBrIWMoEHMpSzsO/kZQBrM3zEw5bNejmyR6GK1xRtoZ11CjMlBxuC6V07hnMilh/XFHhqaagpxITmkisWZSBcNHmRUl/GjHRerQffYWctQwi+52dKkUU2CVlotgOxs3ZKKnvcgu3B9vxAU2DH4guwXNG8GKG5L2OKvP4leQo1NI6V5s5dw3jj1WMAmDdQa/HTg+I1UcuhRcNHnNmSg0tzFeRSrBbk9l1DwmCsRaH5yrEe/MZbrsbrXzHa1P3lsJrSFBRtwaPt8xQczxezA4bzKbGbr7islQP/0C6UG2sKslHQNQJdIyJ81JMJs4bqaQpAKDbnUuEEs+miE4nRJ5ExdegawXzZEedXyyjw1zUTZDXJQnN8AeaPUXEWV6egaUS8lqX2PuKshVi7GYSPagnNQDiM5/jkguhKeueuIXG/tegpEELws6/a0XCzwzH0xU9rW82s/VewDvB9IBbSXzKeT0VNwXA+hcsFG34wVDxlhGEU0WbCMqCR5DqFnlj9gaGRSPiI01Mn+wgIjULWMpCReu80EpoJCaevxSevxeGLLE91TUuZTfHGaIauQdcIbM9bVEWzfM5LCh+tEqG5WeK9j+TiNf6e330V2007HhUC7R07Q6OwFl5nO+Cfn/WgKSijsApop9Ds+hS6zo2CBc+nmC7arPWDpQuDwQuPUqaGlKFXpaTOl10RWuJYuhYJH3EaeQq8qjmX0iM7x0aeAsCnr7mJQ3ZkuFfAi+Kypi7SLJMWcD4a02nggVS/Fgu6RmqmxtZjtQjNzWIEXVJ5m3X52g8FlfD/7vbt4hg3vjuGcyKEuRbDR0uBG/z1kH209l/BOsBvY/jI86noUTMcCLmTCzYuzpcx2pMKOzoG4aO0UT3IHmDFa1yE5shCY59kMJaiKQD1q5nlxy7YoadgaDXqFIIdmggfWbooyEpagNlQHr+hBxJnMGshY+pLGrnIF46lGpWVxtA0eD5NrOV449Vj+Nw778Crdg+J/lV8gA8hBHcEIaS1YPzagSU8hbW/pKrsow5DKevS2Q5PgQZFcLKmALA+NBdmytg6mBUhAB4GSpmaGP4iM192q8JC8kLGtQhDIw2/CFxQzll6ZOfIO6/WI2PpKNoebI/C0mvPv+UL7XSBeQoZM/R+6nkKSS0c6rFtKIvRi/WzpmrBjUJ6jczxNTQCx/Ph+j4ICfUDgL2Wm68YBMAa6x0en48Y/H93xxVIm/qSZxqsNfh7ux6yj5RR6DA8/74dngJ/LCNmFCYXKjg/W8LtOwfD8BE3CoYGS9ci2UeeT7FQcas9hWDhNDQiQkb5tNFwgeOtD7KpxXsKuRQzCo7nN9xdpw1dDI3py5oi4yppYeLekdMgqynOL9+9B7/w6h1N3bfqOfXa4azVCG9zYXvJk+k4Wwa4UQiXk5u2DeCmbQMrcZqrAqUpKNoGNwbtaHPhBo/BNYWRwCicnCpgvuxiY38mFJrtsHsoD6VwuMHoTVdrCgCCFgnsw19vnCaHawe8TiF+vB5Zi7VTcDy/YdVxytQxuVDB3rE8btjSH6akJoWPjFj4qEmjkLUMjPakm7pvHEJI5NqtdnRNgxMIzWaNsB0AbBlgbdW7xStIQmgK68BTWPuvYI3DRyW3I3zkxjyF3owBS9fw/FnWIG9jX1rqdsp20dxTkDUF3uKit4anYBmaWGgbZR4BodDMso/Y3xESFatrkbNkT6H+x5WLze983S5oGhGiXybBcFmGjkqkeG1lwjkpXVszcXZTJ/CCiuakNhgcXhHdzUZBeQqKtiE8hXaEj4JQiB4IzYQQDOUtPHeOGYXN/ZnI6ECAfYhTMaF5rhRtgcHhYqwl9eRvlHkEQMxs6EkbIsQwkLUiMepaZFNs0I7t0oYhnqylY8tABm+5bpN4bfx4HD7TuVFL7nZjGdqaCR/xlFTbo3UN8lbhKXRvNHo9ZR9177u4ShCaQhvqFFy/etc7nE/h+cAobOzPQNcISFCXoBF233hKarwZHof3vpE9hWYKfAZzFv74gRtw1+5hsUAP1GmZLcM8BbcpTeE333IN8mlD7GprFa+xxzVQtN2weK3J8FGrpAytY6M4F4upa2LITr1rf92WfmwZyODKDUubibwe4L2zVupztJwoo9BheNioHZPXuIGRd+DDQYaPRoCxIEXV1Fi2UcrQRZy7KA1MiTfL40Q0Bat5owAA99/ABuxRSkFIc3oCwEI/RdtDxfUaho/u3D0c+b1WmwuAeQ8T85VFtc5uB/K1W+2Ihnh+fU9hQ18aj7/vB1fwzFYf68lTWPuvYI2zHEKzoUU9BQAY602LHTRPS+U7aTkldXKhIg3YqR0+WoynIEMIQcbUmzYKPAd+ruQueuGuVdEMROsfeAuPlaAvYzbtJXUaM5inYHt+pJpZUc160hSUp9BhhKfQDqMQ0xSAsICNz84FQqPBP8BcaJ6Yr+D23/0Gtg8x4bBWSqocPmpGU4gznE9hU3+mqftmg8efLTmLjvuP9qRw7zUbcNuOoarb+JCYZsJS7eSPH7hxDWkKGihlPaJMTe0f66GK1xRto51CM9cUkjwFeREWuxoz/L/isnbWnk9xbKIAIMFT4EZBD9Mq601dq8Vf/fxt6FuEpgAwo7Cpf3GpoKau4WP/7ubE27IpXXRUXanQEQBsH86t2HO1CvcOyo7XdMV3t2KqlFRFuwiF5uXVFGSjwG9PxzwFWVdIyqe3EjyFevOZa7FtKNtUOioQZrTMFO22Lt55y4DtsoE860EcXA745qLseJEBO4pqUmKjtTa8wHooT6HDiDqFtngK1WMTeQHbJil8FPcUeHUvr114zw/thZYQY+ePaxkaBnImspaOK4aWd+fLO28W7MZC82LgYamZorOinsJagmtQJcdrqkixm5G96LWOeqc7jN9GoTn0FMIP5s6RPFKGhldu6RPHeFggZYTiWEXyFN5w1Riu2thb9fjyB78nbeKpD9y97AVL8uO31VNIha22VWgkGe4plByvac+uW7EMDYSsXBHkcqKMQofhWoJPebrm0j9USdlHG/rSOPRb90QqUvntPDwU9xRq7QoNKXwELE1kXixyQZTVxsU7DEspT6EWQlOwvboVzQqWybfU7rmrDWUUOoycdeRToJWNBh+wHk+vjH+hzVimBE9J5f2QsjV0AksKH60UsoFqr6fAHne6aDedHtttyJ6CtQ52wMvJ2+/YnpjlthZR5r/NfOX5C7jt/3u0amhNLeSso1ZDSEmeQhJh+EgP/uejLFl9Qi1PIW5MVgLZQLVVU7D4/AVnXbQ7Xg64uFx2fCU0N2DrYBZ3B3Ov1zrqnW4zxycLuDhXEX39G+FFPIXWjEJS9lES/AsuUlKDRfFywQYhYVFbHJ52t5Ji2nJ5Cjz0tdIpqWsJvnkoOd6K9YZSdB71TrcZ3sOf9w9qhC/1PIp7ChXXw6X5ctPPLTyFBoucGfMULOEp2MhZtecjmDFNYSVIm0zAA9DWEIash6wHcXA5kL2Deq2zFesLZRTaDDcKc2W3wT0ZkfBRzFP4+OMncM+HvgXapAfhJRSvJSE8hdjOf7ro1M0m6oSmQAhBNhDE2+spLE9Yaj0he5yqzUX3oL4NbabssIV5rklPIRI+inkKxycKmC46mK80Z2DCNhfNaQpy9hHARNd6GUVhSurKFujwmoJ2hjDksNR6yC1fDmQPShnO7kG9022GC8zzTXoKfh2h+dJ8BQAwXbCbeqwwfNTIU6iuUwCYUajnKXQifASErS7auTCx9EG0/XHXE/LmQl2j7kG9022GewrNagqyIYiHjyYCo3BZMgq/99XD+Og3jyU+VvPZR9UVzQAwU3DqVq6aHQgfAWFNQTuznjRNCkspETUR2RAo3aV7UN+GNhMKzU16CpHwUfS2JKPwrZcn8J2jk4mPFWoKzQnN6ZjQPF9xa9YosL/j4aOVXSB4/L/dCxMPlanwUTJRTUFdo25BvdNtpuwu0lOoITS7no+pQrVRcDxfGJ44TWsKNVJSgdo1CkDnwkfcU2h3CIMXsLWzUno9EdEUVPZR16CMQpsR2UelJrOPagjNlws2uI2YLoZGwfUoKm7y7E6vWU2hRkoqUH/4uml0yii0X1MAwsI4FS9PRu6hpa5R96DaXLSZyiI9hVpCMxeZAeCyVAhnez6MGtXSbpPFa3xgipi8Jn3h62UfhXNoVzj7yFqeME9umTyQ9YKhwkddiXqn20xl0ZpC+LMcPppYkI1C+HM9T4H3PmqkKdRqcwE08BQ6lX3Ed/RtDvNwA6iMQjJGJCVVhY+6BeUptJnFCs2yIZDDRxNzzBD0pIyIp+B4PiiSi9ma9RRqpaQC9T2F9aYphEKzWvCSMFT4qCtRRqHNLLZ4za+Rkso9hb0beiKaguP58GjyIuYlDNlJgocC4sVrQH1PYSSY9zwa/L9SLEedwnI+7nrBUBXNXYn6NrSZxRav1eqSOjFfQU/awMa+dKR4zfEoKk6N8FGznkJsyI7VZPbRVRt78eT735A4gGc5yS5T6mhuGSql1xOGqmjuSpSn0GZaa3MRHr80X8ZITwqDOQtTklFwfR+eTxMH8ojso0Z1ClrUU4hoCg1mLm+QxnquFMpT6AzR8JHyFLoF9W1oI5RSlANPYaHiVvUySsKvUacwMV/BaE8KA1kLsyUHrueDUgrHo/Ap8xjicE+hUUr5Uj2FTpGxlrl4TXkKiShPoTtp6Z0mhPwOIeQ5QsizhJBHCCGbpNveTwg5Sgh5iRByj3T8ZkLI88FtHybrYX5dgO35oBToz5qgFGKSWT28Gq2zJ+YrGOlJYyjPpoLNlByx6ANIHOLjej4MjTQcCWjG2lywv2G3LffM5aXQG8wH5p5Nu8gqobkuEU1BDdnpGlp9p/+AUnodpfQGAP8I4NcBgBByNYAHAFwD4F4AHyGE8G/0RwE8CGBP8O/eFs+h7bhecsy+ETx0NJJnQmwzuoJfY8jOpfkKRvLMUwBYMZsjnVdSWqrn04Z6AiDNaA6yjgghIl6/EnOXF8tdu4fxh//6erxyc19bHzevitfqYqjeR11JS98GSumc9GsOELmS9wN4mFJaoZSeAHAUwK2EkI0AeimlT1A2JOBTAN7ayjm0mzOXi7jq17+KF87PNb5zDL5751k6zRiFJKG5UHFRtD2M9qbE/GBmFGRPodoouD5taoG7YWs/7to9jJ50aAB4KGk1egqmruHHb94Crc2tFpYr1XW9YKguqV1Jy9tCQsgHAbwdwCyA1weHNwN4Urrb2eCYE/wcP17rsR8E8yqwbdu2Vk+1Kc7PlOB4FKemCrh60+KybHhWUGgUGovNSW0uuDHpTZvCU5iOeQpJ/Y+a9RRu2zmE23ZGh4xbhg7AXZWewnKRV8VrdVFDdrqTht8GQsijhJCDCf/uBwBK6QcopVsB/DWAd/M/S3goWud4IpTShyilt1BKbxkZGWn8atoA340vNDnYRoYv1Dx81EwGUpLQzBd/QydCU7hctEXDOwCJaamu7zdsm12L1ewpLBe83oJ7Y4oo8mdJdZLtHhpuCymldzf5WH8D4J8A/AaYB7BVum0LgPPB8S0Jx1cNtscW9qKd3F+I8+knT0EjwNtuu0IcK1d5Cs0IzdXhI0+ai9CfZSLr5YW4prB0TyGJ0Ch0j6ewZ6wHj/7Ka7BrJN/pU1mVEEKgawSeT1Xvoy6i1eyjPdKv9wE4HPz8JQAPEEJShJAdYILy9ymlFwDME0JuD7KO3g7gi62cQ7uxg1h9o8yhz+8/iy8+G7VnfKEe7eWeQu3HeM/fHcC3j0xEw0eBpxBOUNOQMnRkLR0zJScWPqr2FByPLtlTsAwNaVNbslFZq+we7WmYrdXN8M/TUj9XirVHq9vC/0kIuRKAD+AUgHcAAKX0ECHkMwBeAOACeBellG9t3wngLwFkAHwl+LdqsIMQTbFS31Mo2R5IEHX4/P6z+N7xy3jL9SwjdzhfX1OglOKzT5/FSE8KA4EnAITpqbKnALBdfMX1YkJzDU9hibFfy9BWZY2CorMYGkEFqpajm2hpFaCU/nid2z4I4IMJx/cBuLaV511OmvUUyo4nQi5PHp/CPz13AW+8egwA0JcxYeqkZviIL/qu50Oub+PHuUfAd+2WocF2/YYpqa5PRbXyYrF0rWE1s6L7YGEjT3kKXYQy/zG4UWjkKZQdX4R5XJ+iYHtCnE6bOnrTZk1PQf67pPBRtaegN2UUPN9fuqZgKk9BUQ3/DKoMre5BrQIx+MLbyFMoOZ7IBuL/j8+VAbCisFzKwEINT0EYBY9Gu6T61ZoCEHgKnh+paE5KSXW9pQvNN18xiLlSc/2aFN0DT0VVRqF7UEYhhggfNUhJLTke3KCDHV/Mx2cDo2Ay0dauURntcWPi+9F5CjQMKwHhLs3Sg/CR2zh8tNR88l95494l/Z1ifcPbW6g6he5Bmf8YtvAUaoePfJ/CdsOdO/cuLgaeQsrQkTL0mi2uncCYODU8Bf6/rClUXB+O3PsoyVPwaWSurkLRKspT6D7UOx1DaAp1wke8EyoPG/FF/ELgKaRMLcgYquEpSEJzUpsLN6YpCKPQwFPwWiheUyiS0IWmoD5X3YIyCjG4p1BPaC4FXoTIFpLCR4SwFNKUqSXG/YG40BwerxKaeTfTIPvIlQYuJHoKLdQpKBRJ8Gw25Sl0D+qdjtFMSmo5uE+oKbD/JxYqSBkaCCEsfFTDU+CagevRaJuL4O5ODU3BbtAQz2tBU1AoktBV8VrXoYxCDGcRnkKoKYQ7/JQRTjNLKjCT/45PUeN4tLamYHt+pKV3LaFZaQqKdmLqpKkZHYr1g1pBYsieApV28TI8LBTXFACWeQSgrqbA/87xYnUKMU2Bx3GTi9eShGalKSjai64RFTrqMtS7HYMbBZ8m9xcCJKPg8zBQeL9w7nHt7CPxd74Pn1Ix9aw6+yiqKXCPJG1qiefWSp2CQpGEoWsqJNllKKMQQ64tqKUrlJyo0CwXlfFpZimzTvhIKnrzpME4frx1dix8xI/nUyYqrgffpxFvxvOV0KxoL4ZGVNvsLkO92zFsKeRTS1fgmoLjsUVZnnPQVPhIMiY+peJLV7NOQWdtLvjz5FPMC/mDR17Cjb/zdfzN907DD1pmqBbHinaiPIXuQ1U0x0jyFFzPx3zZxUAwjKUsLfY+RSRVlAvNabOZ7CMfvh9qB15V6+yopsDPLZ82UHF9vHB+DjNFB7/2heeRS+msoll5Coo2YmhEVDUrugP1bseQxVxewPbwU2fwuv/1TbGYl6VqZ9eP9iRKSZ6C59OI3sDxpKwlj0rho6ohO9HeR9yLyacMlB0PkwsV3LV7GACbLd3KkB2FIglDI6ptdpeh3u0YtuuLL0EhCB9dmitjtuSI30tS4ZjrxcNHodAMRL0K8TdSSqovaQrxOgVdmqfAn1fXCDKBFzK1YGNDXxq6RlByPDieyj5StJehfApDalxpV6HCRzFs18dA1sTFuYrwFHjF8oLtoi9rRo2CTyPhI2EUAo+h4nhiQHz4N774W+YpRMNH8dbZXHNYqLgwNFYYV3Y8TBUqGM6nkDF1FG1PeQqKtvOBH7kq0l5Fsf5RnkIM26Poz7Cd0ULgGfAvBW+FXY54CqwAjQ9/57t6/n9ikVks+0jTCDRSXacgawoAUKy4sHTWQmNioQLHoxjOW8hYOkq2pzQFRdvJpwyhpSm6A2UUYtiuh4EcG5HJPQW+SPMhOrKn4PlULM6AnH3EPAbZKHz14AWUHU+ap8DqFDQ+IL3KUwg1Bfb8HgydIG3omCmy2QfD+RSylo6S46nsI4VC0TJqBYlhez4GsmyB5xqCGLwTGIWo0Mx2+3wuc1pqcwGElcdnLhfxjr/aj68dGg/bYwR/qxMCjZDQUwiej2/6efioaLswA0+BI4ePVEWzQqFoFWUUYjguRU/agEZCT6HKKEjVxK7HNAVhFKo0BXZf7l2UbC+Skur5gKYFnoIUPpL7zfDHKlQCoyBlgwxJ4SOlKSgUilZRQnMM22PZRznLEJ4C1wDmE8JHrs+KyobzKRgaQW+GXdJ4+IgbFsen0GLFa7oG6CQaPpILhmSh2dSJeGyAGYWspaNou3BU62yFQtEiyijEsF0flq4jm9Krso8KiUaBwvUp8mkDDz94O/Zu6AFQHT7ifYscNwzxcKFZJwSaRiJCs1wwJKfI5tOG0C0IAQazFjKmjsl5GwBUl1SFQtESyijEsD0fpkGYp2BHs4/C8FFoFHh4yNAIbtk+KI7zMBIPNQlPwfNheoFRCBriifCRNKNZDgMJo2CzqmruKQxkLRi6hoxlYL7MhGfVkkChULSC2lZKUMpmL6d0DbmUIYwAryuYTzIKgScQX4yrPAU3rE2Q5zDIQjMvXounlvLHKtoeCx8FngLPeMqaujg3FT5SKBStoIyCBA/xWIaGrKULo8AnnsnhI76TL0uegozQFJxQSwAQaWzHntMPPIVom4uopqCL46auiQynoRwTtzOWLtJlldCsUChaQRkFCR7iMQNPocgnrInsI/Z72fGRs3h4KPAUYrF8kX3EhWZJcJZ7JVVcHzohEaG5lqbAnkfyFHpCo8A7aCtPQaFQtIIyChK84VzcU+A7e1G8ZnvoSbMCt3Kz4SNJU5Cb5Nku0w8iQnMNTYH/zB+b96TJmmE2kq6K1xQKRQuoFUSCt6a2DC3S+pofl9tc9KSN4GcePop5CvGUVElHkD0F2/WrheZ4+CjuKQSPzTWFjKVHblcoFIqlooyCBPcUTF2DqWvCGHChmc9XKDsecilD/AxUL8Z8IReaQiR8JM9a9qGToE5B1hQShGZ+bqHQHIaPOMooKBSKVlBGQYIbgZShwdKJCPnI4SNKKUpS51NhFGLhIzbwnCSHjyKagheGjyRPQa+hKZi6ht4gdDXWlwYAZGWjoFJSFQpFCyijIMEXbkvXxLQzIBo+sj0fPoUIH/HwUFLWT8rQqyqaXY/Ck7KPKo7PGuJJnoIbm4sgz8g1dYJrNvXi//zMD+C1e0YAABlZU1DFawqFogVU8ZpEPHwU9xQKFVdoCKGm4Im/iZMyNHE7T3e1456Cx4yCpsXqFBLaXABsZi4hBK9/xag4lrHCt1GFjxQKRSuobaWEnH1kGRocj8KXRmoWbE+0voiHj5I9Ba2691FMU+DZR7oGET6KawpaEIoCko2PHD5SdQoKhaIVlFGQkLOP+OJre74oXgOAqQXWYyifYnH9ivAuqhdjOYNJ9jrk4jUg6JJKol1S44s79xaSnkcOHylPQaFQtELXGYWK6+GnP/F9HDw3W3WbHD7iGT98Z88X48mFCgAgn457CtWX0jI0VILb7RrhIwDQCaJCs+dXpbhysTl+HIhlH6k6BYVC0QJdt4JcmqvgsZcnsP/0dNVt3CikZE8haEvRHwzemQw8hZ5UtE7BTAofSZ6CW6N4Daj2FOJtLoDQKJhG9fNkVUqqQqFoE11nFHiIyHb9qtvk3keW8BRoMI2NhYtqewq1NIV4SipN8BS40BwdsiMjjEKSp2AqTUGhULSHrjMKvJiskmAUbC/MJIp6Cn7oKcwHRoF7CsHjJIVtokJzGBry4kYh8BRkobm2ptAgfKSMgkKhaIG2GAVCyHsIIZQQMiwdez8h5Cgh5CVCyD3S8ZsJIc8Ht32Y8JmTKwT3FBKNQiz7CGC9jXwK9GeYp3B+tgQA6A1+r1XRDAR1CrF5CrZHhYHgxMdxOgmaAm9tkVScZumaMCLKU1AoFK3QslEghGwF8EYAp6VjVwN4AMA1AO4F8BFCCN/OfhTAgwD2BP/ubfUcFgNf+HlYJ3JbsFibOoEVLL68U+pA4Ck89tIEetMG9o7l2ePUqGgGWKfUckJFs+dHDZIIHwW2op6mYCV4CoQQEUJKEqIVCoWiWdqxgnwIwHsByNvf+wE8TCmtUEpPADgK4FZCyEYAvZTSJyilFMCnALy1DefQNNwYyJrCuZkSfuOLB1EMuqCmdF0swvxYf6ApFGwPr7tyVMw0qNUQDwjCR051+Mj1aaR1BQsfITKOsyp8xLOParSx4CEk1eZCoVC0QktGgRByH4BzlNIDsZs2Azgj/X42OLY5+Dl+fMUIPYXQKHz14Dg++cQpHDw/ByBap8BHcnJNAQDuvnoMmkZASO3W2UC0zYUtC80eRVoyChqJho9cr1po5imySZoCEGYgKU1BoVC0QsM2F4SQRwFsSLjpAwB+DcAPJf1ZwjFa53it534QLNSEbdu2NTrVphCLtGQUzlwuAgBOThYA8PARH4EZVDCnDegaAQHw2r2s55ChkQaaQph95HqhcXB9HxlLx1yZT0tjhqFWQzygfvEaEGYgKU1BoVC0QkOjQCm9O+k4IeSVAHYAOBBoxVsA7CeE3ArmAWyV7r4FwPng+JaE47We+yEADwHALbfcUtN4LAY7wSicnQ6NgkZYJpEZ7Mz5tDVTI8inDFyzqRd9gchsaGF2UVL2UbSiORo+SktppHGh2ZOK5ThWA09BhI+UpqBQKFpgyQ3xKKXPAxBd2QghJwHcQimdJIR8CcDfEEL+CMAmMEH5+5RSjxAyTwi5HcD3ALwdwJ+08gIWC9+5y0Lz2WmWUTRfcZE2o4Iu9xRMXcOvvekV2DPWI/6uGU/Bdn1QSqN1Ch6N1BaIhni0GU2hfvhIV5qCQqFogWXpkkopPUQI+QyAFwC4AN5FKeWr8DsB/CWADICvBP9WjLinQCkV4SMg3IkLodkONYMfvzkawjJ0gvlKndbZ0pxmuQ2361Ok5IKzoHW2X0dT4EbKahA+UpqCQqFohbYZBUrp9tjvHwTwwYT77QNwbbued7FUYkLzdNERYjIQCrqWEJrdyO8yctw/qdJYHsnpSjUInu8jbYSXPj6Ok6WkNt/7CAjbZyujoFAoWqHrAtBxoZl7CUM5ll0kBF2Rkso9hepLJS/ASWEbbmAqrifCR5Syc0jHPAWNEPDyBdf3E7KP9Mh5xcmqOgWFQtEGum4FiXsKZwKR+dYdgwDCRTfuKSSlnMrHamkKAGutIQvbJdsT2gXAso90jXkIvk/h0+pwVNj7qH6dgtIUFApFK3SdUYhrClxkvi0wCmHsPuopJIWPZEOQaBRMHj7yIk3wSo4XFZql8BG/X82GeDU8hYyqU1AoFG2g64xCPPvozOUi+rMm9m5gWUWinYQR8xQSFls5pFSrSyrAqp4dqV122fFEOAiQw0dUpKXGw1UpoSkkL/pjPSnkU4YyCgqFoiW6bkZz3FM4M13C1oEsNvdnAITZR7xOoGQ31hQMjSCprx/XDcqOB0cKHzkehWkQaAQiVBR6Cn7ksTn1uqQCwE/dtg13Xz2mhuwoFIqW6LoVJN7m4uzlIrYOZrChLw1CovUAGgEWKvWyjwKjUCOOz2sHSo4H26ORxzA0TSzgWuApeJKnUFNTqLHopwwdWwayjV6+QqFQ1KXrjEI8+2h8rowNvRmkDB0j+ZQI0wBsAZbrFOLwRb1mmmjgKRRtD67vI5uKzj3gojH3FHw/bKtdU1NQQrJCoVhGus4oyJ6C71MUbU9MUfuB7YPYNZIX97WM0CgkLcZGA0+Bi788fJSVxGVDlzwFKXxUS1PYNZLHUM7CUD61+BetUCgUTdJ1moJone35KAYtKnLB4v2nb7spcl9L1yJtLuLokqaQBA8fFW0PjkerJqRxQyPXKXBNIR4+unXHIJ7+729cxCtVKBSKxdN9noKUBTRbcgBEB9/LyJ5CkoDLF/Va4aOsyWxu0fbg+D5yqdAG85AR+zmoU5A9BZVFpFAoOkDXGQU+9AYApgs2gLBFRBzZO0gKH/E2F7XaVactdvtC2QWlUeNj6kQYEy3ofeRJmoJqga1QKDpB1xkF2VOYKTJPIVfHU+Ak9TbiQnEt8ZfPTg49EtlT0MTf8S6pQDi2s1aWkUKhUCwnXbfyRDyFIvcUko2CvDAnicm6lD2UBCEEWVPHXNmpeh5TJyIkxcZxssfgQrjyFBQKRSfoOqMQ9RSYUcjWCB9ZsfTUONxQ1NvVpy0dc6Vqj0TXiNANNC30FPj5KU1BoVB0gq4zChXHQz4QfKeLDYRmyTtINAoNNAX+2EnhI5aSGmYf8cdQnoJCoegkXWcUbM9HT1CXMNPIKASeAiHJi3RYp1D7MmbM0CjEU1JDowIRPuIps0pTUCgUnaDrVp6KIxuF+uEjs0G/IUOvX6cAMEMwX2a1DvHwUZLQrDwFhULRSbrPKHg+etImgFBolttPyIgmdDUWaJ6SWs8oZCVNQTY+ckoqE5qD83OVpqBQKDpHVxkFSilsN/QUhKZg1sg+kprjJdGozQXAwkfzQVO9bMRTCDUFTVOagkKhWB10lVHgmT1caJ4tObCkHkRxUk2Hj+poCpJ3EElJlbKPdCl8xD0FpSkoFIpO0FUrD9+Fy+GjWqEjoHq2Qhx5nkItZC8kZ0XbXKg6BYVCsdroKqPAd+G96dBTqBU6AhrPMBCts+uFjyTvINrmIrmiWdUpKBSKTtJVRiH0FJhRoBTIpmo3ijUbLPqhp1AvfBQagpQZL16r7SmoCWoKhaITdNXKU4mFj4DaNQqA5CnUWPQbTV4DouGjlKGJjCZDJ2HxmhY+Fq9TUJ6CQqHoBF1lFOKeAhBOR0uCVzSbRvICbUqaQC0yNUJGhqaJhZ+oOgWFQrFK6CqjwHfheSlklKsTPhLzmht4CrU8CSBWxRxrgid+JgnhI2UUFApFB+gqo8AX3Iylix17rQ6pQPPZR3q98JH0+JauRR4zOqM5OEdPaQoKhaJzdNXKwzUFSw9j+y1lHwlPoX7xGsfUNRGSkj0FLRjHCYStvVX4SKFQdIKuMgrcU0iZusgEqhc+CrOPaoSPRCioueI1OXxkyl1SpYrmikpJVSgUHaSrjALXFGRPoV74KMw+qiE0N5i8BtSuTZDnKegalNCsUChWBV1mFLinoCFlNhE+atDmotHkNSAaPopoCppWNaMZCI2CanOhUCg6QVetPImaQjPZRzU8ARFeajollYjH1HUS8RridQrKUVAoFJ2gq4yCneQpNJF9ZDXwFOplCmVrjOA0YkLzYM4CAJyfKcPQCAhRVkGhUKw8XWUURPhI10NPoQlNoVGbi2bCR5augRAS8S7kv98xnAMhwOnLRaUnKBSKjtFVRiHiKRhssa41dQ0IBeSa8xQa1DEAYfiI30cuiJONQtrUsXUgG9y3q94WhUKxiuiq1ceWNQWjCU+hQfgoXNRrX0ZL16CRcGCP3GRvz1gPtg1m0ZdhvZh2j+aDx1OegkKh6AxdZRQqrgdTZ32GUkbzKam1hGS9iZRUQgiyliEyjWTv4FW7h/Gt974e6SDEtGskV/f5FAqFYrmpHTtZh9iuH+7+gwU/Vzd81GAcp95YUwCCthrcgNSpklaegkKh6DRdZRQqri8qmUNNobGnYNUUmuu31uZkTB08mcgSGUfV99s1woyC0hQUCkWnaGn1IYT8JiHkHCHk2eDfm6Tb3k8IOUoIeYkQco90/GZCyPPBbR8mK5h7meQp1A0ftclTyFp6JBXV1JNTTrlRUJ6CQqHoFO3Ykn6IUnpD8O/LAEAIuRrAAwCuAXAvgI8QQvjq+1EADwLYE/y7tw3n0BQV1xPGINVE+KjZlNR6Q3YA3pU18CoMreaiP5CzMJSzlKagUCg6xnKFj+4H8DCltALgBCHkKIBbCSEnAfRSSp8AAELIpwC8FcBXluk88AuffAqnpooAgAuzZWzoSwNgRoEQIG3WzxyS/48TiseNw0eeT8Vj1bv/rtE8pgt23cdTKBSK5aIdRuHdhJC3A9gH4L9QSqcBbAbwpHSfs8ExJ/g5fjwRQsiDYF4Ftm3btqST2zaYEzv+PWN5vHbvCADgrTduxmhvum7lcH/WxH95417cc82GxNv3juXxjtfuwqt2D9U9h1949Q6Ug5bYP37TFpFllMS7Xr9bGQWFQtExCKW0/h0IeRRA0qr4AbCFfxIABfA7ADZSSn+OEPKnAJ6glP5V8BgfB/BlAKcB/C6l9O7g+KsBvJdS+pZGJ3rLLbfQffv2Nf3CFAqFQgEQQp6mlN7S7P0begp8AW/iif8cwD8Gv54FsFW6eQuA88HxLQnHFQqFQrEKaDX7aKP0648COBj8/CUADxBCUoSQHWCC8vcppRcAzBNCbg+yjt4O4IutnINCoVAo2kermsLvE0JuAAsfnQTwHwCAUnqIEPIZAC8AcAG8i1LqBX/zTgB/CSADJjAvm8isUCgUisXRUFNYLShNQaFQKBbPYjUFVTqrUCgUCoEyCgqFQqEQKKOgUCgUCoEyCgqFQqEQrBmhmRAyAeDUEv98GKzIbjWyms8NWN3nt5rPDVjd56fObems5vNLOrcrKKUjzT7AmjEKrUAI2bcY9X0lWc3nBqzu81vN5was7vNT57Z0VvP5tePcVPhIoVAoFAJlFBQKhUIh6Baj8FCnT6AOq/ncgNV9fqv53IDVfX7q3JbOaj6/ls+tKzQFhUKhUDRHt3gKCoVCoWgCZRQUCoVCIVjXRoEQci8h5CVCyFFCyK+ugvPZSgj5F0LIi4SQQ4SQXwqO/yYh5Bwh5Nng35s6dH4nCSHPB+ewLzg2SAj5OiHkSPD/QIfO7Urp+jxLCJkjhPxyp64dIeQThJBLhJCD0rGa14oQ8v7gc/gSIeSeDp3fHxBCDhNCniOEfIEQ0h8c304IKUnX8GMdOLea7+NKXrsa5/a30nmdJIQ8Gxxf0esWPGetNaR9nz1K6br8B0AHcAzATgAWgAMAru7wOW0EcFPwcw+AlwFcDeA3AbxnFVyzkwCGY8d+H8CvBj//KoDfWwXnqQMYB3BFp64dgNcAuAnAwUbXKniPDwBIAdgRfC71DpzfDwEwgp9/Tzq/7fL9OnTtEt/Hlb52SecWu/0PAfx6J65b8Jy11pC2ffbWs6dwK4CjlNLjlFIbwMMA7u/kCVFKL1BK9wc/zwN4EXVmVK8S7gfwyeDnTwJ4a+dORfAGAMcopUutcG8ZSum3AFyOHa51re4H8DCltEIpPQHgKNjnc0XPj1L6CKXUDX59EtEpiCtGjWtXixW9dvXOLRgM9hMA/u9yPX8j6qwhbfvsrWejsBnAGen3s1hFCzAhZDuAGwF8Lzj07sCt/0SnQjRgw5IeIYQ8TQh5MDg2RtnEPAT/j3bo3GQeQPSLuRquHVD7Wq3Gz+LPITrgagch5BlCyGOEzU7vBEnv42q6dq8GcJFSekQ61rHrFltD2vbZW89GgSQcWxX5t4SQPIDPAfhlSukcgI8C2AXgBgAXwFzUTvAqSulNAH4YwLsIIa/p0HnUhBBiAbgPwN8Fh1bLtavHqvosEkI+ADYR8a+DQxcAbKOU3gjgVwD8DSGkd4VPq9b7uJqu3U8huhnp2HVLWENq3jXhWN3rt56NwlkAW6XftwA436FzERBCTLA3868ppZ8HAErpRUqpRyn1Afw5ljm0UAtK6fng/0sAvhCcx0USzOIO/r/UiXOT+GEA+ymlF4HVc+0Cal2rVfNZJIT8NIA3A3gbDYLOQWhhKvj5abC4896VPK867+OquHaEEAPAjwH4W36sU9ctaQ1BGz9769koPAVgDyFkR7C7fADAlzp5QkFM8uMAXqSU/pF0fKN0tx8FcDD+tytwbjlCSA//GUyUPAh2zX46uNtPA/jiSp9bjMhubTVcO4la1+pLAB4ghKQIITsA7AHw/ZU+OULIvQDeB+A+SmlROj5CCNGDn3cG53d8hc+t1vu4Kq4dgLsBHKaUnuUHOnHdaq0haOdnbyWV85X+B+BNYOr8MQAfWAXncxeY6/YcgGeDf28C8GkAzwfHvwRgYwfObSdYlsIBAIf49QIwBOAbAI4E/w928PplAUwB6JOOdeTagRmmCwAcsN3Yz9e7VgA+EHwOXwLwwx06v6Ng8WX+2ftYcN8fD97zAwD2A3hLB86t5vu4ktcu6dyC438J4B2x+67odQues9Ya0rbPnmpzoVAoFArBeg4fKRQKhWKRKKOgUCgUCoEyCgqFQqEQKKOgUCgUCoEyCgqFQqEQKKOgUCgUCoEyCgqFQqEQ/D83o84OHNPxwwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
